From 4160a8a2485ed59c9d55e94784beda0e88252e62 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 19 Aug 2016 23:17:27 -0700 Subject: [PATCH 001/114] Revert "Revert "Make promote_op rely on Core.Inference.return_type"" This reverts commit dbdaf72b22078aa692875806b5ba20c8e7154e77. ref #17389 and #17929 --- base/abstractarray.jl | 12 ++++----- base/arraymath.jl | 48 ++++++++++++++++++---------------- base/bitarray.jl | 33 +++++++++++++++-------- base/broadcast.jl | 4 +-- base/char.jl | 4 --- base/complex.jl | 3 ++- base/dates/arithmetic.jl | 18 ------------- base/float.jl | 2 ++ base/int.jl | 3 +++ base/irrationals.jl | 7 ----- base/linalg/matmul.jl | 31 +++++++++++----------- base/number.jl | 12 --------- base/pkg/resolve/fieldvalue.jl | 1 - base/promotion.jl | 39 ++++++++++++++++++++++----- doc/devdocs/promote-op.rst | 36 ------------------------- test/arrayops.jl | 2 -- test/broadcast.jl | 19 +++++++++++--- test/linalg/matmul.jl | 4 +-- test/numbers.jl | 34 ++++++++++++------------ 19 files changed, 144 insertions(+), 168 deletions(-) delete mode 100644 doc/devdocs/promote-op.rst diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 74dc06ccccf68..ef2b9fa100be3 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1653,12 +1653,12 @@ end # These are needed because map(eltype, As) is not inferrable promote_eltype_op(::Any) = (@_pure_meta; Bottom) -promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T)) -promote_eltype_op{T}(op, ::T ) = (@_pure_meta; promote_op(op, T)) -promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S)) -promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::S) = (@_pure_meta; promote_op(op, R, S)) -promote_eltype_op{R,S}(op, ::R, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S)) -promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_op(op, eltype(A), promote_eltype_op(op, B, C, D...))) +promote_eltype_op(op, A) = (@_pure_meta; _promote_op(op, eltype(A))) +promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; _promote_op(op, T)) +promote_eltype_op{T}(op, ::AbstractArray{T}, A) = (@_pure_meta; _promote_op(op, T, eltype(A))) +promote_eltype_op{T}(op, A, ::AbstractArray{T}) = (@_pure_meta; _promote_op(op, eltype(A), T)) +promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; _promote_op(op, R, S)) +promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_eltype_op(op, promote_eltype_op(op, A, B), C, D...)) ## 1 argument map!{F}(f::F, A::AbstractArray) = map!(f, A, A) diff --git a/base/arraymath.jl b/base/arraymath.jl index 60b78f0a2fed3..aa2f30b152b28 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -35,30 +35,26 @@ function !(A::AbstractArray{Bool}) end ## Binary arithmetic operators ## -@pure promote_array_type{S<:Number, A<:AbstractArray}(F, ::Type{S}, ::Type{A}) = - promote_array_type(F, S, eltype(A), promote_op(F, S, eltype(A))) -@pure promote_array_type{S<:Number, A<:AbstractArray}(F, ::Type{A}, ::Type{S}) = - promote_array_type(F, S, eltype(A), promote_op(F, eltype(A), S)) - -@pure promote_array_type{S, A, P}(F, ::Type{S}, ::Type{A}, ::Type{P}) = P -@pure promote_array_type{S<:Real, A<:AbstractFloat, P}(F, ::Type{S}, ::Type{A}, ::Type{P}) = A -@pure promote_array_type{S<:Integer, A<:Integer, P}(F::typeof(./), ::Type{S}, ::Type{A}, ::Type{P}) = P -@pure promote_array_type{S<:Integer, A<:Integer, P}(F::typeof(.\), ::Type{S}, ::Type{A}, ::Type{P}) = P -@pure promote_array_type{S<:Integer, A<:Integer, P}(F, ::Type{S}, ::Type{A}, ::Type{P}) = A -@pure promote_array_type{S<:Integer, P}(F::typeof(./), ::Type{S}, ::Type{Bool}, ::Type{P}) = P -@pure promote_array_type{S<:Integer, P}(F::typeof(.\), ::Type{S}, ::Type{Bool}, ::Type{P}) = P -@pure promote_array_type{S<:Integer, P}(F, ::Type{S}, ::Type{Bool}, ::Type{P}) = P + +promote_array_type(F, ::Type, ::Type, T::Type) = T +promote_array_type{S<:Real, A<:AbstractFloat}(F, ::Type{S}, ::Type{A}, ::Type) = A +promote_array_type{S<:Integer, A<:Integer}(F, ::Type{S}, ::Type{A}, ::Type) = A +promote_array_type{S<:Integer, A<:Integer}(::typeof(./), ::Type{S}, ::Type{A}, T::Type) = T +promote_array_type{S<:Integer, A<:Integer}(::typeof(.\), ::Type{S}, ::Type{A}, T::Type) = T +promote_array_type{S<:Integer}(::typeof(./), ::Type{S}, ::Type{Bool}, T::Type) = T +promote_array_type{S<:Integer}(::typeof(.\), ::Type{S}, ::Type{Bool}, T::Type) = T +promote_array_type{S<:Integer}(F, ::Type{S}, ::Type{Bool}, T::Type) = T for f in (:+, :-, :div, :mod, :&, :|, :$) - @eval ($f){S,T}(A::AbstractArray{S}, B::AbstractArray{T}) = - _elementwise($f, A, B, promote_eltype_op($f, A, B)) + @eval ($f)(A::AbstractArray, B::AbstractArray) = + _elementwise($f, promote_op($f, eltype(A), eltype(B)), A, B) end -function _elementwise{S,T}(op, A::AbstractArray{S}, B::AbstractArray{T}, ::Type{Any}) - promote_shape(A,B) # check size compatibility +function _elementwise(op, ::Type{Any}, A::AbstractArray, B::AbstractArray) + promote_shape(A, B) # check size compatibility return broadcast(op, A, B) end -function _elementwise{S,T,R}(op, A::AbstractArray{S}, B::AbstractArray{T}, ::Type{R}) - F = similar(A, R, promote_shape(A,B)) +function _elementwise(op, T::Type, A::AbstractArray, B::AbstractArray) + F = similar(A, T, promote_shape(A, B)) for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) @inbounds F[iF] = op(A[iA], B[iB]) end @@ -67,15 +63,21 @@ end for f in (:.+, :.-, :.*, :./, :.\, :.^, :.÷, :.%, :.<<, :.>>, :div, :mod, :rem, :&, :|, :$) @eval begin - function ($f){T}(A::Number, B::AbstractArray{T}) - F = similar(B, promote_array_type($f,typeof(A),typeof(B))) + function ($f)(A::Number, B::AbstractArray) + P = promote_op($f, typeof(A), eltype(B)) + T = promote_array_type($f, typeof(A), eltype(B), P) + T === Any && return [($f)(A, b) for b in B] + F = similar(B, T) for (iF, iB) in zip(eachindex(F), eachindex(B)) @inbounds F[iF] = ($f)(A, B[iB]) end return F end - function ($f){T}(A::AbstractArray{T}, B::Number) - F = similar(A, promote_array_type($f,typeof(A),typeof(B))) + function ($f)(A::AbstractArray, B::Number) + P = promote_op($f, eltype(A), typeof(B)) + T = promote_array_type($f, typeof(B), eltype(A), P) + T === Any && return [($f)(a, B) for a in A] + F = similar(A, T) for (iF, iA) in zip(eachindex(F), eachindex(A)) @inbounds F[iF] = ($f)(A[iA], B) end diff --git a/base/bitarray.jl b/base/bitarray.jl index 72272422c7f20..3e67d3c86e911 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -1052,18 +1052,29 @@ for f in (:+, :-) return r end end + for (f) in (:.+, :.-) - for (arg1, arg2, T, fargs) in ((:(B::BitArray), :(x::Bool) , Int , :(b, x)), - (:(B::BitArray), :(x::Number) , :(promote_array_type($f, BitArray, typeof(x))), :(b, x)), - (:(x::Bool) , :(B::BitArray), Int , :(x, b)), - (:(x::Number) , :(B::BitArray), :(promote_array_type($f, typeof(x), BitArray)), :(x, b))) + for (arg1, arg2, T, t) in ((:(B::BitArray), :(x::Bool) , Int , (:b, :x)), + (:(B::BitArray), :(x::Number) , :(Bool, typeof(x)), (:b, :x)), + (:(x::Bool) , :(B::BitArray), Int , (:x, :b)), + (:(x::Number) , :(B::BitArray), :(typeof(x), Bool), (:x, :b))) @eval function ($f)($arg1, $arg2) - r = Array{$T}(size(B)) + $(if T === Int + quote + r = Array{Int}(size(B)) + end + else + quote + T = promote_op($f, $(T.args[1]), $(T.args[2])) + T === Any && return [($f)($(t[1]), $(t[2])) for b in B] + r = Array{T}(size(B)) + end + end) bi = start(B) ri = 1 while !done(B, bi) b, bi = next(B, bi) - @inbounds r[ri] = ($f)($fargs...) + @inbounds r[ri] = ($f)($(t[1]), $(t[2])) ri += 1 end return r @@ -1095,9 +1106,8 @@ function div(x::Bool, B::BitArray) end function div(x::Number, B::BitArray) all(B) || throw(DivideError()) - pt = promote_array_type(div, typeof(x), BitArray) y = div(x, true) - reshape(pt[ y for i = 1:length(B) ], size(B)) + return fill(y, size(B)) end function mod(A::BitArray, B::BitArray) @@ -1116,15 +1126,16 @@ function mod(x::Bool, B::BitArray) end function mod(x::Number, B::BitArray) all(B) || throw(DivideError()) - pt = promote_array_type(mod, typeof(x), BitArray) y = mod(x, true) - reshape(pt[ y for i = 1:length(B) ], size(B)) + return fill(y, size(B)) end for f in (:div, :mod) @eval begin function ($f)(B::BitArray, x::Number) - F = Array{promote_array_type($f, BitArray, typeof(x))}(size(B)) + T = promote_op($f, Bool, typeof(x)) + T === Any && return [($f)(b, x) for b in B] + F = Array{T}(size(B)) for i = 1:length(F) F[i] = ($f)(B[i], x) end diff --git a/base/broadcast.jl b/base/broadcast.jl index 937d66375bd9a..e42f2a67edfbc 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -3,7 +3,7 @@ module Broadcast using Base.Cartesian -using Base: promote_op, promote_eltype, promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, tail, OneTo, to_shape +using Base: promote_eltype_op, @get!, _msk_end, unsafe_bitgetindex, linearindices, tail, OneTo, to_shape import Base: .+, .-, .*, ./, .\, .//, .==, .<, .!=, .<=, .÷, .%, .<<, .>>, .^ export broadcast, broadcast!, bitbroadcast, dotview export broadcast_getindex, broadcast_setindex! @@ -299,7 +299,7 @@ end ## elementwise operators ## for op in (:÷, :%, :<<, :>>, :-, :/, :\, ://, :^) - @eval $(Symbol(:., op))(A::AbstractArray, B::AbstractArray) = broadcast($(op), A, B) + @eval $(Symbol(:., op))(A::AbstractArray, B::AbstractArray) = broadcast($op, A, B) end .+(As::AbstractArray...) = broadcast(+, As...) .*(As::AbstractArray...) = broadcast(*, As...) diff --git a/base/char.jl b/base/char.jl index 52aa1492e2f86..58481c6d39c2a 100644 --- a/base/char.jl +++ b/base/char.jl @@ -40,10 +40,6 @@ hash(x::Char, h::UInt) = hash_uint64(((UInt64(x)+hashchar_seed)<<32) $ UInt64(h) +(x::Char, y::Integer) = Char(Int32(x) + Int32(y)) +(x::Integer, y::Char) = y + x -Base.promote_op{I<:Integer}(::typeof(-), ::Type{Char}, ::Type{I}) = Char -Base.promote_op{I<:Integer}(::typeof(+), ::Type{Char}, ::Type{I}) = Char -Base.promote_op{I<:Integer}(::typeof(+), ::Type{I}, ::Type{Char}) = Char - bswap(x::Char) = Char(bswap(UInt32(x))) print(io::IO, c::Char) = (write(io, c); nothing) diff --git a/base/complex.jl b/base/complex.jl index da644f9412331..7be18a036f951 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -804,7 +804,8 @@ big{T<:AbstractFloat,N}(A::AbstractArray{Complex{T},N}) = convert(AbstractArray{ ## promotion to complex ## -promote_array_type{S<:Union{Complex, Real}, AT<:AbstractFloat, P}(F, ::Type{S}, ::Type{Complex{AT}}, ::Type{P}) = Complex{AT} +_default_type(T::Type{Complex}) = Complex{Int} +promote_array_type{S<:Union{Complex, Real}, T<:AbstractFloat}(F, ::Type{S}, ::Type{Complex{T}}, ::Type) = Complex{T} function complex{S<:Real,T<:Real}(A::AbstractArray{S}, B::AbstractArray{T}) if size(A) != size(B); throw(DimensionMismatch()); end diff --git a/base/dates/arithmetic.jl b/base/dates/arithmetic.jl index 4a94fc13df911..94752ffe2390a 100644 --- a/base/dates/arithmetic.jl +++ b/base/dates/arithmetic.jl @@ -94,21 +94,3 @@ end # AbstractArray{TimeType}, AbstractArray{TimeType} (-){T<:TimeType}(x::OrdinalRange{T}, y::OrdinalRange{T}) = collect(x) - collect(y) (-){T<:TimeType}(x::Range{T}, y::Range{T}) = collect(x) - collect(y) - -# promotion rules - -for op in (:+, :-, :.+, :.-) - @eval begin - Base.promote_op{P<:Period}(::typeof($op), ::Type{P}, ::Type{P}) = P - Base.promote_op{P1<:Period,P2<:Period}(::typeof($op), ::Type{P1}, ::Type{P2}) = CompoundPeriod - Base.promote_op{D<:Date}(::typeof($op), ::Type{D}, ::Type{D}) = Day - Base.promote_op{D<:DateTime}(::typeof($op), ::Type{D}, ::Type{D}) = Millisecond - end -end - -for op in (:/, :%, :div, :mod, :./, :.%) - @eval begin - Base.promote_op{P<:Period}(::typeof($op), ::Type{P}, ::Type{P}) = typeof($op(1,1)) - Base.promote_op{P<:Period,R<:Real}(::typeof($op), ::Type{P}, ::Type{R}) = P - end -end diff --git a/base/float.jl b/base/float.jl index 20b7e9a2513ca..01767dd3d4123 100644 --- a/base/float.jl +++ b/base/float.jl @@ -230,6 +230,8 @@ promote_rule(::Type{Float64}, ::Type{Float32}) = Float64 widen(::Type{Float16}) = Float32 widen(::Type{Float32}) = Float64 +_default_type(T::Union{Type{Real},Type{AbstractFloat}}) = Float64 + ## floating point arithmetic ## -(x::Float32) = box(Float32,neg_float(unbox(Float32,x))) -(x::Float64) = box(Float64,neg_float(unbox(Float64,x))) diff --git a/base/int.jl b/base/int.jl index ca3242cc682f5..79cfb0ddafb0b 100644 --- a/base/int.jl +++ b/base/int.jl @@ -305,6 +305,9 @@ promote_rule{T<:BitSigned64}(::Type{UInt64}, ::Type{T}) = UInt64 promote_rule{T<:Union{UInt32, UInt64}}(::Type{T}, ::Type{Int128}) = Int128 promote_rule{T<:BitSigned}(::Type{UInt128}, ::Type{T}) = UInt128 +_default_type(T::Type{Unsigned}) = UInt +_default_type(T::Union{Type{Integer},Type{Signed}}) = Int + ## traits ## typemin(::Type{Int8 }) = Int8(-128) diff --git a/base/irrationals.jl b/base/irrationals.jl index e76f40760c0b9..a5e091f5ca1e3 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -10,13 +10,6 @@ promote_rule{s}(::Type{Irrational{s}}, ::Type{Float32}) = Float32 promote_rule{s,t}(::Type{Irrational{s}}, ::Type{Irrational{t}}) = Float64 promote_rule{s,T<:Number}(::Type{Irrational{s}}, ::Type{T}) = promote_type(Float64,T) -promote_op{S<:Irrational,T<:Irrational}(op::Any, ::Type{S}, ::Type{T}) = - promote_op(op, Float64, Float64) -promote_op{S<:Irrational,T<:Number}(op::Any, ::Type{S}, ::Type{T}) = - promote_op(op, Float64, T) -promote_op{S<:Irrational,T<:Number}(op::Any, ::Type{T}, ::Type{S}) = - promote_op(op, T, Float64) - convert(::Type{AbstractFloat}, x::Irrational) = Float64(x) convert(::Type{Float16}, x::Irrational) = Float16(Float32(x)) convert{T<:Real}(::Type{Complex{T}}, x::Irrational) = convert(Complex{T}, convert(T,x)) diff --git a/base/linalg/matmul.jl b/base/linalg/matmul.jl index 6c9117a3cd06e..aa713321a42bb 100644 --- a/base/linalg/matmul.jl +++ b/base/linalg/matmul.jl @@ -76,11 +76,11 @@ At_mul_B{T<:BlasComplex}(x::StridedVector{T}, y::StridedVector{T}) = [BLAS.dotu( # Matrix-vector multiplication function (*){T<:BlasFloat,S}(A::StridedMatrix{T}, x::StridedVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_B!(similar(x, TS, size(A,1)), A, convert(AbstractVector{TS}, x)) end function (*){T,S}(A::AbstractMatrix{T}, x::AbstractVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_B!(similar(x,TS,size(A,1)),A,x) end (*)(A::AbstractVector, B::AbstractMatrix) = reshape(A,length(A),1)*B @@ -99,22 +99,22 @@ end A_mul_B!(y::AbstractVector, A::AbstractVecOrMat, x::AbstractVector) = generic_matvecmul!(y, 'N', A, x) function At_mul_B{T<:BlasFloat,S}(A::StridedMatrix{T}, x::StridedVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) At_mul_B!(similar(x,TS,size(A,2)), A, convert(AbstractVector{TS}, x)) end function At_mul_B{T,S}(A::AbstractMatrix{T}, x::AbstractVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) At_mul_B!(similar(x,TS,size(A,2)), A, x) end At_mul_B!{T<:BlasFloat}(y::StridedVector{T}, A::StridedVecOrMat{T}, x::StridedVector{T}) = gemv!(y, 'T', A, x) At_mul_B!(y::AbstractVector, A::AbstractVecOrMat, x::AbstractVector) = generic_matvecmul!(y, 'T', A, x) function Ac_mul_B{T<:BlasFloat,S}(A::StridedMatrix{T}, x::StridedVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) Ac_mul_B!(similar(x,TS,size(A,2)),A,convert(AbstractVector{TS},x)) end function Ac_mul_B{T,S}(A::AbstractMatrix{T}, x::AbstractVector{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) Ac_mul_B!(similar(x,TS,size(A,2)), A, x) end @@ -142,14 +142,14 @@ end A_mul_B!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'N', 'N', A, B) function At_mul_B{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) - TS = promote_op(*,arithtype(T), arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) At_mul_B!(similar(B, TS, (size(A,2), size(B,2))), A, B) end At_mul_B!{T<:BlasFloat}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = is(A,B) ? syrk_wrapper!(C, 'T', A) : gemm_wrapper!(C, 'T', 'N', A, B) At_mul_B!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'T', 'N', A, B) function A_mul_Bt{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) - TS = promote_op(*,arithtype(T), arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_Bt!(similar(B, TS, (size(A,1), size(B,1))), A, B) end A_mul_Bt!{T<:BlasFloat}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = is(A,B) ? syrk_wrapper!(C, 'N', A) : gemm_wrapper!(C, 'N', 'T', A, B) @@ -166,7 +166,7 @@ end A_mul_Bt!(C::AbstractVecOrMat, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'N', 'T', A, B) function At_mul_Bt{T,S}(A::AbstractMatrix{T}, B::AbstractVecOrMat{S}) - TS = promote_op(*,arithtype(T), arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) At_mul_Bt!(similar(B, TS, (size(A,2), size(B,1))), A, B) end At_mul_Bt!{T<:BlasFloat}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = gemm_wrapper!(C, 'T', 'T', A, B) @@ -175,7 +175,7 @@ At_mul_Bt!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generi Ac_mul_B{T<:BlasReal}(A::StridedMatrix{T}, B::StridedMatrix{T}) = At_mul_B(A, B) Ac_mul_B!{T<:BlasReal}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = At_mul_B!(C, A, B) function Ac_mul_B{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) - TS = promote_op(*,arithtype(T), arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) Ac_mul_B!(similar(B, TS, (size(A,2), size(B,2))), A, B) end Ac_mul_B!{T<:BlasComplex}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = is(A,B) ? herk_wrapper!(C,'C',A) : gemm_wrapper!(C,'C', 'N', A, B) @@ -184,13 +184,14 @@ Ac_mul_B!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic A_mul_Bc{T<:BlasFloat,S<:BlasReal}(A::StridedMatrix{T}, B::StridedMatrix{S}) = A_mul_Bt(A, B) A_mul_Bc!{T<:BlasFloat,S<:BlasReal}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{S}) = A_mul_Bt!(C, A, B) function A_mul_Bc{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) - TS = promote_op(*,arithtype(T),arithtype(S)) + TS = promote_op(*, arithtype(T), arithtype(S)) A_mul_Bc!(similar(B,TS,(size(A,1),size(B,1))),A,B) end A_mul_Bc!{T<:BlasComplex}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = is(A,B) ? herk_wrapper!(C, 'N', A) : gemm_wrapper!(C, 'N', 'C', A, B) A_mul_Bc!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'N', 'C', A, B) -Ac_mul_Bc{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) = Ac_mul_Bc!(similar(B, promote_op(*,arithtype(T), arithtype(S)), (size(A,2), size(B,1))), A, B) +Ac_mul_Bc{T,S}(A::AbstractMatrix{T}, B::AbstractMatrix{S}) = + Ac_mul_Bc!(similar(B, promote_op(*, arithtype(T), arithtype(S)), (size(A,2), size(B,1))), A, B) Ac_mul_Bc!{T<:BlasFloat}(C::StridedMatrix{T}, A::StridedVecOrMat{T}, B::StridedVecOrMat{T}) = gemm_wrapper!(C, 'C', 'C', A, B) Ac_mul_Bc!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'C', 'C', A, B) Ac_mul_Bt!(C::AbstractMatrix, A::AbstractVecOrMat, B::AbstractVecOrMat) = generic_matmatmul!(C, 'C', 'T', A, B) @@ -423,7 +424,7 @@ end function generic_matmatmul{T,S}(tA, tB, A::AbstractVecOrMat{T}, B::AbstractMatrix{S}) mA, nA = lapack_size(tA, A) mB, nB = lapack_size(tB, B) - C = similar(B, promote_op(*,arithtype(T),arithtype(S)), mA, nB) + C = similar(B, promote_op(*, arithtype(T), arithtype(S)), mA, nB) generic_matmatmul!(C, tA, tB, A, B) end @@ -617,7 +618,7 @@ end # multiply 2x2 matrices function matmul2x2{T,S}(tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) - matmul2x2!(similar(B, promote_op(*,T,S), 2, 2), tA, tB, A, B) + matmul2x2!(similar(B, promote_op(*, T, S), 2, 2), tA, tB, A, B) end function matmul2x2!{T,S,R}(C::AbstractMatrix{R}, tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) @@ -646,7 +647,7 @@ end # Multiply 3x3 matrices function matmul3x3{T,S}(tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) - matmul3x3!(similar(B, promote_op(*,T,S), 3, 3), tA, tB, A, B) + matmul3x3!(similar(B, promote_op(*, T, S), 3, 3), tA, tB, A, B) end function matmul3x3!{T,S,R}(C::AbstractMatrix{R}, tA, tB, A::AbstractMatrix{T}, B::AbstractMatrix{S}) diff --git a/base/number.jl b/base/number.jl index d2d3270f1cfc4..42ffb708736ee 100644 --- a/base/number.jl +++ b/base/number.jl @@ -63,16 +63,4 @@ zero{T<:Number}(::Type{T}) = convert(T,0) one(x::Number) = oftype(x,1) one{T<:Number}(::Type{T}) = convert(T,1) -promote_op{R,S<:Number}(::Type{R}, ::Type{S}) = (@_pure_meta; R) # to fix ambiguities -function promote_op{T<:Number}(op, ::Type{T}) - S = typeof(op(one(T))) - # preserve the most general (abstract) type when possible - return isleaftype(T) ? S : typejoin(S, T) -end -function promote_op{R<:Number,S<:Number}(op, ::Type{R}, ::Type{S}) - T = typeof(op(one(R), one(S))) - # preserve the most general (abstract) type when possible - return isleaftype(R) && isleaftype(S) ? T : typejoin(R, S, T) -end - factorial(x::Number) = gamma(x + 1) # fallback for x not Integer diff --git a/base/pkg/resolve/fieldvalue.jl b/base/pkg/resolve/fieldvalue.jl index a3666388d35a7..fe175ce356b0b 100644 --- a/base/pkg/resolve/fieldvalue.jl +++ b/base/pkg/resolve/fieldvalue.jl @@ -42,7 +42,6 @@ Base.typemin(::Type{FieldValue}) = (x=typemin(Int); y=typemin(VersionWeight); Fi Base.:-(a::FieldValue, b::FieldValue) = FieldValue(a.l0-b.l0, a.l1-b.l1, a.l2-b.l2, a.l3-b.l3, a.l4-b.l4) Base.:+(a::FieldValue, b::FieldValue) = FieldValue(a.l0+b.l0, a.l1+b.l1, a.l2+b.l2, a.l3+b.l3, a.l4+b.l4) -Base.promote_op(::Union{typeof(+), typeof(-)}, ::Type{FieldValue}, ::Type{FieldValue}) = FieldValue function Base.isless(a::FieldValue, b::FieldValue) a.l0 < b.l0 && return true diff --git a/base/promotion.jl b/base/promotion.jl index 778de13c004c9..449dae5add81a 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -217,13 +217,38 @@ max(x::Real, y::Real) = max(promote(x,y)...) min(x::Real, y::Real) = min(promote(x,y)...) minmax(x::Real, y::Real) = minmax(promote(x, y)...) -# "Promotion" that takes a function into account. You can override this -# as needed. For example, if you need to provide a custom result type -# for the multiplication of two types, -# promote_op{R<:MyType,S<:MyType}(::typeof(*), ::Type{R}, ::Type{S}) = MyType{multype(R,S)} -promote_op(::Any) = (@_pure_meta; Bottom) -promote_op(::Any, ::Any, ::Any...) = (@_pure_meta; Any) -promote_op{T}(::Type{T}, ::Any) = (@_pure_meta; T) +# "Promotion" that takes a function into account. These are meant to be +# used mainly by broadcast methods, so it is advised against overriding them +if isdefined(Core, :Inference) + function _promote_op(op, T::Type) + G = Tuple{Generator{Tuple{T},typeof(op)}} + R = Core.Inference.return_type(first, G) + return isleaftype(R) ? R : Any + end + function _promote_op(op, R::Type, S::Type) + F = typeof(a -> op(a...)) + G = Tuple{Generator{Zip2{Tuple{R},Tuple{S}},F}} + T = Core.Inference.return_type(first, G) + return isleaftype(T) ? T : Any + end +else + _promote_op(::Any...) = (@_pure_meta; Any) +end +_default_type(T::Type) = (@_pure_meta; T) + +promote_op(::Any...) = (@_pure_meta; Any) +promote_op(T::Type, ::Any) = (@_pure_meta; T) +promote_op(T::Type, ::Type) = (@_pure_meta; T) # To handle ambiguities +# Promotion that tries to preserve non-concrete types +function promote_op(f, S::Type) + T = _promote_op(f, _default_type(S)) + return isleaftype(S) ? T : typejoin(S, T) +end +function promote_op(f, R::Type, S::Type) + T = _promote_op(f, _default_type(R), _default_type(S)) + isleaftype(R) && return isleaftype(S) ? T : typejoin(S, T) + return isleaftype(S) ? typejoin(R, T) : typejoin(R, S, T) +end ## catch-alls to prevent infinite recursion when definitions are missing ## diff --git a/doc/devdocs/promote-op.rst b/doc/devdocs/promote-op.rst deleted file mode 100644 index 53a5a93768549..0000000000000 --- a/doc/devdocs/promote-op.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. currentmodule:: Base - -.. _devdocs-promote-op: - -Operator-sensitive promotion -============================ - -In certain cases, the :ref:`simple rules for promotion -` may not be sufficient. For example, consider a -type that can represent an object with physical units, here restricted -to a single unit like "meter":: - - immutable MeterUnits{T,P} <: Number - val::T - end - MeterUnits{T}(val::T, pow::Int) = MeterUnits{T,pow}(val) - - m = MeterUnits(1.0, 1) # 1.0 meter, i.e. units of length - m2 = MeterUnits(1.0, 2) # 1.0 meter^2, i.e. units of area - -Now let's define the operations ``+`` and ``*`` for these objects: -``m+m`` should have the type of ``m`` but ``m*m`` should have the type -of ``m2``. When the result type depends on the operation, and not -just the input types, ``promote_rule`` will be inadequate. - -Fortunately, it's possible to provide such definitions via ``promote_op``:: - - Base.promote_op{R,S}(::typeof(+), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),1} - Base.promote_op{R,S}(::typeof(*), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),2} - Base.promote_op{R,S}(::typeof(.*), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),2} - -The first one defines the promotion rule for ``+``, and the second one -for ``*``. - -It's worth noting that as julia's internal representation of functions -evolves, this interface may change in a future version of Julia. diff --git a/test/arrayops.jl b/test/arrayops.jl index 984128a58816a..5280484c95faf 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -1439,7 +1439,6 @@ b = rand(6,7) @test_throws ArgumentError copy!(a,2:3,1:3,b,1:5,2:7) @test_throws ArgumentError Base.copy_transpose!(a,2:3,1:3,b,1:5,2:7) -# return type declarations (promote_op) module RetTypeDecl using Base.Test import Base: +, *, .*, convert @@ -1457,7 +1456,6 @@ module RetTypeDecl (*){T}(x::MeterUnits{T,1}, y::MeterUnits{T,1}) = MeterUnits{T,2}(x.val*y.val) (.*){T}(x::MeterUnits{T,1}, y::MeterUnits{T,1}) = MeterUnits{T,2}(x.val*y.val) convert{T,pow}(::Type{MeterUnits{T,pow}}, y::Real) = MeterUnits{T,pow}(convert(T,y)) - Base.promote_op{R,S}(::typeof(*), ::Type{MeterUnits{R,1}}, ::Type{MeterUnits{S,1}}) = MeterUnits{promote_type(R,S),2} @test @inferred(m+[m,m]) == [m+m,m+m] @test @inferred([m,m]+m) == [m+m,m+m] diff --git a/test/broadcast.jl b/test/broadcast.jl index f14788413f05a..7cc1ceffa500f 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -225,12 +225,11 @@ let x = sin.(1:10) @test sin.(atan2.(x, 3.7)) == broadcast(x -> sin(atan2(x,3.7)), x) @test atan2.(x, 3.7) == broadcast(x -> atan2(x,3.7), x) == broadcast(atan2, x, 3.7) end -# Use side effects to check for loop fusion. Note that, due to #17314, -# a broadcasted function is currently called an extra time with an argument 1. +# Use side effects to check for loop fusion. let g = Int[] - f17300(x) = begin; push!(g, x); x+1; end + f17300(x) = begin; push!(g, x); x+2; end f17300.(f17300.(f17300.(1:3))) - @test g == [1,2,3, 1,2,3, 2,3,4, 3,4,5] + @test g == [1,3,5, 2,4,6, 3,5,7] end # fusion with splatted args: let x = sin.(1:10), a = [x] @@ -295,3 +294,15 @@ end let foo = [[1,2,3],[4,5,6],[7,8,9]] @test max.(foo...) == broadcast(max, foo...) == [7,8,9] end + +# Issue 17314 +@test broadcast(x->log(log(log(x))), [1000]) == [log(log(log(1000)))] +let f17314 = x -> x < 0 ? false : x + @test eltype(broadcast(f17314, 1:3)) === Int + @test eltype(broadcast(f17314, -1:1)) === Integer + @test eltype(broadcast(f17314, Int[])) === Union{} +end +let io = IOBuffer() + broadcast(x->print(io,x), 1:5) # broadcast with side effects + @test takebuf_array(io) == [0x31,0x32,0x33,0x34,0x35] +end diff --git a/test/linalg/matmul.jl b/test/linalg/matmul.jl index 44bb9a6afa85b..4755f1d9ddd64 100644 --- a/test/linalg/matmul.jl +++ b/test/linalg/matmul.jl @@ -320,10 +320,10 @@ end immutable RootInt i::Int end -import Base: *, transpose, promote_op +import Base: *, transpose (*)(x::RootInt, y::RootInt) = x.i*y.i transpose(x::RootInt) = x -promote_op(::typeof(*), ::Type{RootInt}, ::Type{RootInt}) = Int +@test Base.promote_op(*, RootInt, RootInt) === Int a = [RootInt(3)] C = [0] diff --git a/test/numbers.jl b/test/numbers.jl index 8ef819c0685ae..a0ccc138e776f 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2791,21 +2791,21 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Complex{Int}, Complex{UInt}, Complex32, Complex64, Complex128) for S in types for op in (+, -) - T = @inferred Base.promote_op(op, S) + T = @inferred Base._promote_op(op, S) t = @inferred op(one(S)) @test T === typeof(t) end - end - - @test @inferred(Base.promote_op(!, Bool)) === Bool - for R in types, S in types - for op in (+, -, *, /, ^) - T = @inferred Base.promote_op(op, R, S) - t = @inferred op(one(R), one(S)) - @test T === typeof(t) + for R in types + for op in (+, -, *, /, ^) + T = @inferred Base._promote_op(op, S, R) + t = @inferred op(one(S), one(R)) + @test T === typeof(t) + end end end + + @test @inferred(Base._promote_op(!, Bool)) === Bool end let types = (Base.BitInteger_types..., BigInt, Bool, @@ -2813,23 +2813,23 @@ let types = (Base.BitInteger_types..., BigInt, Bool, Float16, Float32, Float64, BigFloat) for S in types, T in types for op in (<, >, <=, >=, (==)) - @test @inferred(Base.promote_op(op, S, T)) === Bool + @test @inferred(Base._promote_op(op, S, T)) === Bool end end end let types = (Base.BitInteger_types..., BigInt, Bool) for S in types - T = @inferred Base.promote_op(~, S) + T = @inferred Base._promote_op(~, S) t = @inferred ~one(S) @test T === typeof(t) - end - for S in types, T in types - for op in (&, |, <<, >>, (>>>), %, ÷) - T = @inferred Base.promote_op(op, S, T) - t = @inferred op(one(S), one(T)) - @test T === typeof(t) + for R in types + for op in (&, |, <<, >>, (>>>), %, ÷) + T = @inferred Base._promote_op(op, S, R) + t = @inferred op(one(S), one(R)) + @test T === typeof(t) + end end end end From b9d7579f62490fa358d8aeacf5c4ba7fd73bdccd Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 19 Aug 2016 23:17:49 -0700 Subject: [PATCH 002/114] Revert "Revert "Restore type parameter for elementwise ops"" This reverts commit e6f5d89f519f95ce71370bb44f62c387388cda35. ref #17798 and #17389 and #17929 --- base/arraymath.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/arraymath.jl b/base/arraymath.jl index aa2f30b152b28..0de1bd39486f5 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -53,7 +53,7 @@ function _elementwise(op, ::Type{Any}, A::AbstractArray, B::AbstractArray) promote_shape(A, B) # check size compatibility return broadcast(op, A, B) end -function _elementwise(op, T::Type, A::AbstractArray, B::AbstractArray) +function _elementwise{T}(op, ::Type{T}, A::AbstractArray, B::AbstractArray) F = similar(A, T, promote_shape(A, B)) for (iF, iA, iB) in zip(eachindex(F), eachindex(A), eachindex(B)) @inbounds F[iF] = op(A[iA], B[iB]) From 19b4222e473f62ba2524e5c25f159cb3cc81c8cc Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Tue, 16 Aug 2016 07:54:23 -0700 Subject: [PATCH 003/114] Reinstate downloading old versions of winrpm gcc dll's This mostly reverts #17906 and puts #15521 back in place, but from a slightly more permanent download location now. Unfortunately using the latest opensuse copy of the gcc dll's is causing issues when a cygwin-built Julia tries to load an opensuse-built libzmq.dll. Using these slightly old gcc 5 versions seems to fix it. I suspect the difference is which libstdc++ ABI is being used. GCC 5 has both old and new available, but many distros had it using the old ABI by default for compatibility. GCC 6 (which opensuse is using now) is more likely to be using the new ABI by default. (cherry picked from commit 477a026055b83ee10879f489f64243d0825b35f9) ref #18059 --- Makefile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8b20b2e4585e8..23b641fe4d45b 100644 --- a/Makefile +++ b/Makefile @@ -468,7 +468,7 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll busybox.exe libexpat-1.dll zlib1.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) + cp 7z.exe 7z.dll busybox.exe libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. @@ -593,7 +593,14 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686)) $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920.exe && \ 7z x -y 7z920.exe 7z.exe 7z.dll && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.2 \ - "mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \ + "mingw32-libexpat1 mingw32-zlib1" && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libgfortran3-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libquadmath0-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libstdc++6-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libgcc_s_sjlj1-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libssp0-5.3.0-1.1.noarch.rpm && \ + for i in *.rpm; do 7z x -y $$i; done && \ + for i in *.cpio; do 7z x -y $$i; done && \ cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll . else ifeq ($(ARCH),x86_64) cd $(JULIAHOME)/dist-extras && \ @@ -602,7 +609,14 @@ else ifeq ($(ARCH),x86_64) mv _7z.dll 7z.dll && \ mv _7z.exe 7z.exe && \ ../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.2 \ - "mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \ + "mingw64-libexpat1 mingw64-zlib1" && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libgfortran3-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libquadmath0-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libstdc++6-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libgcc_s_seh1-5.3.0-1.1.noarch.rpm && \ + $(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libssp0-5.3.0-1.1.noarch.rpm && \ + for i in *.rpm; do 7z x -y $$i; done && \ + for i in *.cpio; do 7z x -y $$i; done && \ cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll . else $(error no win-extras target for ARCH=$(ARCH)) From 4364deda7d8f709b8ed4fd15b8f6de40feddcaa5 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Fri, 5 Aug 2016 14:03:16 -0700 Subject: [PATCH 004/114] Only download busybox-w32 when USE_GPL_LIBS is 1 some of the spawn and cmdlineargs tests may fail in a no-GPL Windows build if executed outside of the build environment (all of which are GPL themselves...), we could either skip them or try to get them to work via calling powershell or some other non-GPL coreutils replacement (toybox? uutils? sbase + now-LPGL cygwin?) (cherry picked from commit 2a9dd3b2ef06e5ba36a1f3d8d33839498b7cc501) ref #17851 --- Makefile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 23b641fe4d45b..6d0bcfbaa40a8 100644 --- a/Makefile +++ b/Makefile @@ -468,7 +468,11 @@ endif ifeq ($(OS), WINNT) [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ - cp 7z.exe 7z.dll busybox.exe libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) + cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) +ifeq ($(USE_GPL_LIBS), 1) + [ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \ + cp busybox.exe $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin ) +endif cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe # create file listing for uninstall. note: must have Windows path separators and line endings. @@ -624,12 +628,15 @@ endif cd $(JULIAHOME)/dist-extras && \ $(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920_extra.7z && \ $(JLDOWNLOAD) https://unsis.googlecode.com/files/nsis-2.46.5-Unicode-setup.exe && \ - $(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-483-g31277ab.exe && \ chmod a+x 7z.exe && \ chmod a+x 7z.dll && \ $(call spawn,./7z.exe) x -y -onsis nsis-2.46.5-Unicode-setup.exe && \ - chmod a+x ./nsis/makensis.exe && \ + chmod a+x ./nsis/makensis.exe +ifeq ($(USE_GPL_LIBS), 1) + cd $(JULIAHOME)/dist-extras && \ + $(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-483-g31277ab.exe && \ chmod a+x busybox.exe +endif # various statistics about the build that may interest the user ifeq ($(USE_SYSTEM_LLVM), 1) From 65aa0b05b281f0b085e831d8ee0c217c1f4d896e Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Wed, 3 Aug 2016 19:05:10 +0530 Subject: [PATCH 005/114] Fix pcre clean target. (cherry picked from commit a434002b698ae6dd7cf26b3fac78d490b4a44f55) ref #17783 --- deps/pcre.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/pcre.mk b/deps/pcre.mk index c66197fc7f50c..5ae612452e763 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -36,7 +36,7 @@ $(PCRE_OBJ_TARGET): $(PCRE_SRC_TARGET) touch -c $@ clean-pcre: - -$(MAKE) -C pcre2-$(PCRE_VER) clean + -$(MAKE) -C $(BUILDDIR)/pcre2-$(PCRE_VER) clean -rm -f $(build_shlibdir)/libpcre* distclean-pcre: -rm -rf $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2 $(SRCDIR)/srccache/pcre2-$(PCRE_VER) $(BUILDDIR)/pcre2-$(PCRE_VER) From 56623061dad10df82d9317c74b3e75aca553ad81 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Wed, 3 Aug 2016 19:07:09 +0530 Subject: [PATCH 006/114] Build curl to provide proxy support for Pkg. The curl build depends on mbedtls and libssh2 Patch for mbedtls to allow curl to build with mbedtls support Add curl to LICENSE.md, README.md and all the other various locations (cherry picked from commit 046c6eb7f896d806de3fc38da2e8c9153b2dba39) ref #17783 --- LICENSE.md | 1 + Make.inc | 1 + Makefile | 3 ++ README.md | 1 + deps/Makefile | 9 ++++- deps/Versions.make | 1 + deps/checksums/curl-7.50.1.tar.bz2/md5 | 1 + deps/checksums/curl-7.50.1.tar.bz2/sha512 | 1 + deps/curl.mk | 46 +++++++++++++++++++++++ deps/mbedtls.mk | 6 ++- deps/patches/mbedtls-ssl.h.patch | 12 ++++++ 11 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/curl-7.50.1.tar.bz2/md5 create mode 100644 deps/checksums/curl-7.50.1.tar.bz2/sha512 create mode 100644 deps/curl.mk create mode 100644 deps/patches/mbedtls-ssl.h.patch diff --git a/LICENSE.md b/LICENSE.md index fd312fcee06f3..ab11841e04e27 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -57,6 +57,7 @@ their own licenses: - [FFTW](http://fftw.org/doc/License-and-Copyright.html) [GPL2+] - [GMP](http://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+] - [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception] +- [CURL](https://curl.haxx.se/docs/copyright.html) [MIT/X derivative] - [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3] - [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0] - [MPFR](http://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+] diff --git a/Make.inc b/Make.inc index 8bd586dd92cb8..568a94cb22cf7 100644 --- a/Make.inc +++ b/Make.inc @@ -38,6 +38,7 @@ USE_SYSTEM_LIBUV:=0 USE_SYSTEM_UTF8PROC:=0 USE_SYSTEM_MBEDTLS:=0 USE_SYSTEM_LIBSSH2:=0 +USE_SYSTEM_CURL:=0 USE_SYSTEM_LIBGIT2:=0 USE_SYSTEM_PATCHELF:=0 diff --git a/Makefile b/Makefile index 6d0bcfbaa40a8..c10090e4b73c2 100644 --- a/Makefile +++ b/Makefile @@ -277,6 +277,9 @@ endif ifeq ($(USE_SYSTEM_LIBSSH2),0) JL_PRIVATE_LIBS += ssh2 endif +ifeq ($(USE_SYSTEM_CURL),0) +JL_PRIVATE_LIBS += curl +endif ifeq ($(USE_SYSTEM_LIBGIT2),0) JL_PRIVATE_LIBS += git2 endif diff --git a/README.md b/README.md index a1d390f724d6d..3aee9f9c68de0 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,7 @@ Julia uses the following external libraries, which are automatically downloaded - **[GMP]** (>= 5.0) — GNU multiple precision arithmetic library, needed for `BigInt` support. - **[MPFR]** (>= 3.0) — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support. - **[libgit2]** (>= 0.23) — Git linkable library, used by Julia's package manager +- **[curl]** (>= 7.50) — libcurl provides download and proxy support for Julia's package manager - **[libssh2]** (>= 1.7) — library for SSH transport, used by libgit2 for packages with SSH remotes - **[mbedtls]** (>= 2.2) — library used for cryptography and transport layer security, used by libssh2 - **[utf8proc]** (>= 2.0) — a library for processing UTF-8 encoded Unicode strings diff --git a/deps/Makefile b/deps/Makefile index 08c3f96fc503e..9d6dd32375950 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -17,7 +17,7 @@ include $(SRCDIR)/llvm-ver.make # additionally all targets should be listed in the getall target for easier off-line compilation # if you are adding a new target, it can help to copy an similar, existing target # -# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv +# autoconf configure-driven scripts: llvm pcre arpack fftw unwind gmp mpfr patchelf libuv curl # custom Makefile rules: openlibm dsfmt suitesparse-wrapper suitesparse lapack openblas utf8proc objconv osxunwind # entirely custom: virtualenv # CMake libs: libgit2 libssh2 mbedtls @@ -171,6 +171,10 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0) STAGE2_DEPS += libssh2 endif +ifeq ($(USE_SYSTEM_CURL), 0) +STAGE2_DEPS += curl +endif + ifeq ($(USE_SYSTEM_LIBGIT2), 0) STAGE3_DEPS += libgit2 endif @@ -222,7 +226,7 @@ install: $(addprefix install-, $(DEP_LIBS)) cleanall: $(addprefix clean-, $(DEP_LIBS)) distcleanall: $(addprefix distclean-, $(DEP_LIBS)) rm -rf $(build_prefix) -getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-libgit2 +getall: get-llvm get-libuv get-pcre get-openlibm get-openspecfun get-dsfmt get-openblas get-lapack get-fftw get-suitesparse get-arpack get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-virtualenv get-objconv get-mbedtls get-libssh2 get-curl get-libgit2 ## PATHS ## # sort is used to remove potential duplicates @@ -271,6 +275,7 @@ include $(SRCDIR)/mpfr.mk include $(SRCDIR)/patchelf.mk include $(SRCDIR)/mbedtls.mk include $(SRCDIR)/libssh2.mk +include $(SRCDIR)/curl.mk include $(SRCDIR)/libgit2.mk include $(SRCDIR)/virtualenv.mk diff --git a/deps/Versions.make b/deps/Versions.make index 4d6c5fc751b79..cdf2c1eb84de9 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -13,3 +13,4 @@ MPFR_VER = 3.1.4 PATCHELF_VER = 0.9 VIRTUALENV_VER = 15.0.0 MBEDTLS_VER = 2.3.0 +CURL_VER = 7.50.1 diff --git a/deps/checksums/curl-7.50.1.tar.bz2/md5 b/deps/checksums/curl-7.50.1.tar.bz2/md5 new file mode 100644 index 0000000000000..111b575728535 --- /dev/null +++ b/deps/checksums/curl-7.50.1.tar.bz2/md5 @@ -0,0 +1 @@ +015f6a0217ca6f2c5442ca406476920b diff --git a/deps/checksums/curl-7.50.1.tar.bz2/sha512 b/deps/checksums/curl-7.50.1.tar.bz2/sha512 new file mode 100644 index 0000000000000..95a02a7febe1c --- /dev/null +++ b/deps/checksums/curl-7.50.1.tar.bz2/sha512 @@ -0,0 +1 @@ +94acd91fcf8ff2605e1ba2d086c2c366257b61eaf516b9ea44e574e315feb5b30f6e47d89051f259e026ef5dd9edde5f7b15a6af9ee6a38f641da354e1e677b1 diff --git a/deps/curl.mk b/deps/curl.mk new file mode 100644 index 0000000000000..12295cde6ad08 --- /dev/null +++ b/deps/curl.mk @@ -0,0 +1,46 @@ +## CURL ## + +CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/.libs/libcurl.$(SHLIB_EXT) +CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT) + +ifneq ($(OS),WINNT) +CURL_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) +endif + +$(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache + $(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2 +$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET) + $(JLCHECKSUM) $< + cd $(dir $<) && $(TAR) jxf $(notdir $<) + touch -c $@ +$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure + mkdir -p $(dir $@) + cd $(dir $@) && \ + $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --with-mbedtls=$(build_shlibdir)/.. CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + touch -c $@ +$(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status + $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) + touch -c $@ +$(BUILDDIR)/curl-$(CURL_VER)/checked: $(CURL_SRC_TARGET) +ifeq ($(OS),$(BUILD_OS)) +ifneq ($(OS),WINNT) + $(MAKE) -C $(dir $@) check -j1 +endif +endif + echo 1 > $@ +$(CURL_OBJ_TARGET): $(CURL_SRC_TARGET) + $(call make-install,curl-$(CURL_VER),$(LIBTOOL_CCLD)) + $(INSTALL_NAME_CMD)libcurl.$(SHLIB_EXT) $@ + touch -c $@ + +clean-curl: + -$(MAKE) -C $(BUILDDIR)/curl-$(CURL_VER) clean + -rm -f $(build_shlibdir)/libcurl* +distclean-curl: + -rm -rf $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(SRCDIR)/srccache/curl-$(CURL_VER) $(BUILDDIR)/curl-$(CURL_VER) + +get-curl: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 +configure-curl: $(BUILDDIR)/curl-$(CURL_VER)/config.status +compile-curl: $(CURL_SRC_TARGET) +check-curl: $(BUILDDIR)/curl-$(CURL_VER)/checked +install-curl: $(CURL_OBJ_TARGET) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 1ed14a48ab8a0..0d9759c46d017 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -35,7 +35,11 @@ $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt: $(SRCDIR)/srccache/$(MBEDTLS_S $(TAR) -C $(dir $@) --strip-components 1 -xf $< touch -c $@ -$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt +$(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied: | $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt + cd $(SRCDIR)/srccache/$(MBEDTLS_SRC)/include/mbedtls && patch -p0 -f < $(SRCDIR)/patches/mbedtls-ssl.h.patch + echo 1 > $@ + +$(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/Makefile: $(SRCDIR)/srccache/$(MBEDTLS_SRC)/CMakeLists.txt $(SRCDIR)/srccache/$(MBEDTLS_SRC)/mbedtls-ssl.h.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(MBEDTLS_OPTS) diff --git a/deps/patches/mbedtls-ssl.h.patch b/deps/patches/mbedtls-ssl.h.patch new file mode 100644 index 0000000000000..11785c5d88c60 --- /dev/null +++ b/deps/patches/mbedtls-ssl.h.patch @@ -0,0 +1,12 @@ +--- ssl.h.old 2016-06-28 18:12:06.000000000 +0530 ++++ ssl.h 2016-08-03 18:51:34.000000000 +0530 +@@ -54,7 +54,8 @@ + #endif + + #if defined(MBEDTLS_HAVE_TIME) +-#include ++//#include ++#include "platform.h" + #endif + + /* From 9a6f03584dca103a5fd586bb6ac2c60bee388050 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Wed, 3 Aug 2016 19:38:25 +0530 Subject: [PATCH 007/114] Have libgit2 use our curl. Enforce curl as a dependency for the libgit2 build. Fix the libgit2 clean target Build curl only on linux and osx. Add pkg-config to build dependencies in README.md. Required to detect curl in libgit2 build. (cherry picked from commit 3687527bfc98be8438d2bf018e87c70ea123403b) ref #17783 --- README.md | 2 ++ contrib/windows/msys_build.sh | 1 + deps/Makefile | 8 +++++--- deps/libgit2.mk | 9 +++++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3aee9f9c68de0..d893f1b32c036 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,7 @@ Building Julia requires that the following software be installed: - **[patch]** — for modifying source code. - **[cmake]** — needed to build `libgit2`. - **[openssl]** — needed for HTTPS support in `libgit2` on Linux, install via `apt-get install libssl-dev` or `yum install openssl-devel`. +- **[pkg-config]** - needed to build libgit2 correctly, especially for proxy support Julia uses the following external libraries, which are automatically downloaded (or in a few cases, included in the Julia source repository) and then compiled from source the first time you run `make`: @@ -326,6 +327,7 @@ For a longer overview of Julia's dependencies, see these [slides](https://github [openssl]: https://www.openssl.org [libssh2]: https://www.libssh2.org [mbedtls]: https://tls.mbed.org/ +[pkg-config]: https://www.freedesktop.org/wiki/Software/pkg-config/ ### System Provided Libraries diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index 2d91f26a6af72..ae343ae1dc85b 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -181,6 +181,7 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user echo 'override STAGE1_DEPS = libuv' >> Make.user echo 'override STAGE2_DEPS = utf8proc' >> Make.user echo 'override STAGE3_DEPS = ' >> Make.user +echo 'override STAGE4_DEPS = ' >> Make.user if [ -n "$USEMSVC" ]; then # Openlibm doesn't build well with MSVC right now diff --git a/deps/Makefile b/deps/Makefile index 9d6dd32375950..37b4defd90884 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -171,12 +171,14 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0) STAGE2_DEPS += libssh2 endif +ifneq ($(OS), WINNT) ifeq ($(USE_SYSTEM_CURL), 0) -STAGE2_DEPS += curl +STAGE3_DEPS += curl +endif endif ifeq ($(USE_SYSTEM_LIBGIT2), 0) -STAGE3_DEPS += libgit2 +STAGE4_DEPS += libgit2 endif ifeq ($(USE_SYSTEM_MPFR), 0) @@ -215,7 +217,7 @@ endif ## Common build target prefixes -DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) +DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) $(STAGE4_DEPS) default: install | $(build_prefix) get: $(addprefix get-, $(DEP_LIBS)) diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 113fbcea7c388..0e530132f7732 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -19,6 +19,8 @@ else LIBGIT2_OPTS += -DBUILD_CLAR=OFF -DDLLTOOL=`which $(CROSS_COMPILE)dlltool` LIBGIT2_OPTS += -DCMAKE_FIND_ROOT_PATH=/usr/$(XC_HOST) -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY endif +else +LIBGIT2_OPTS += -DCURL_INCLUDE_DIRS=$(build_includedir) -DCURL_LIBRARIES="-L$(build_shlibdir) -lcurl" endif ifeq ($(OS),Linux) @@ -44,6 +46,9 @@ $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied: | $(SR ifeq ($(OS),Linux) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied endif +ifneq ($(OS),WINNT) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(CURL_OBJ_TARGET) +endif $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied $(LIBSSH2_OBJ_TARGET) mkdir -p $(dir $@) cd $(dir $@) && \ @@ -73,14 +78,14 @@ ifeq ($(OS),Linux) for LIB in libssl libcrypto; do \ LIB_PATH=$$(echo "$$LIBGIT_LIBS" | grep "$$LIB"); \ echo "LIB_PATH for $$LIB: $$LIB_PATH"; \ - [ ! -z "$$LIB_PATH" ] && cp -v "$$LIB_PATH" $(build_shlibdir); \ + [ ! -z "$$LIB_PATH" ] && cp -v -f "$$LIB_PATH" $(build_shlibdir); \ done endif touch -c $@ clean-libgit2: -$(MAKE) -C $(BUILDDIR)/$(LIBGIT2_SRC_DIR) clean - -rm -f $(LIBGIT2_OBJ_TARGET) + -rm -f $(build_shlibdir)/libgit2* $(build_shlibdir)/libssl* $(build_shlibdir)/libcrypto* get-libgit2: $(LIBGIT2_SRC_FILE) configure-libgit2: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile From a36da57dbd46dc423c06ea1cbfc21f8984ca1908 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Sat, 6 Aug 2016 14:32:52 +0530 Subject: [PATCH 008/114] Fix mbedtls clean target to delete all installed files. (cherry picked from commit 11a8e07d89d26fe5c42a3cbcfbfbcf85a5d790aa) ref #17783 --- deps/mbedtls.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 0d9759c46d017..150a39f2a1196 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -71,7 +71,7 @@ endif clean-mbedtls: -$(MAKE) -C $(BUILDDIR)/mbedtls-$(MBEDTLS_VER) clean - -rm -f $(MBEDTLS_OBJ_TARGET) + -rm -f $(MBEDTLS_OBJ_TARGET) $(build_shlibdir)/libmbed* distclean-mbedtls: -rm -rf $(SRCDIR)/srccache/$(MBEDTLS_SRC).tgz \ From 05197a51fa015dfcbec05ddba9f2e2e2a2945150 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 9 Aug 2016 17:47:48 -0500 Subject: [PATCH 009/114] Support non-1 indices and fix type problems in DFT (fixes #17896) (cherry picked from commit 996e2757cb39b88f112e03f602339aa2b7f0ee65) ref #17919 --- base/dft.jl | 39 ++++++++++++++------ base/multidimensional.jl | 78 ++++++++++++++++++++++++++++++++++++++++ test/fft.jl | 8 +++++ test/offsetarray.jl | 33 ++++++++++++++++- 4 files changed, 147 insertions(+), 11 deletions(-) diff --git a/base/dft.jl b/base/dft.jl index 04b9c5bf7875b..700bbc68c2dec 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -20,22 +20,41 @@ export fft, ifft, bfft, fft!, ifft!, bfft!, plan_fft, plan_ifft, plan_bfft, plan_fft!, plan_ifft!, plan_bfft!, rfft, irfft, brfft, plan_rfft, plan_irfft, plan_brfft -complexfloat{T<:AbstractFloat}(x::AbstractArray{Complex{T}}) = x +typealias FFTWFloat Union{Float32,Float64} +fftwfloat(x) = _fftwfloat(float(x)) +_fftwfloat{T<:FFTWFloat}(::Type{T}) = T +_fftwfloat(::Type{Float16}) = Float32 +_fftwfloat{T}(::Type{T}) = error("type $T not supported") +_fftwfloat{T}(x::T) = _fftwfloat(T)(x) + +complexfloat{T<:FFTWFloat}(x::StridedArray{Complex{T}}) = x +realfloat{T<:FFTWFloat}(x::StridedArray{T}) = x # return an Array, rather than similar(x), to avoid an extra copy for FFTW # (which only works on StridedArray types). -complexfloat{T<:Complex}(x::AbstractArray{T}) = copy!(Array{typeof(float(one(T)))}(size(x)), x) -complexfloat{T<:AbstractFloat}(x::AbstractArray{T}) = copy!(Array{typeof(complex(one(T)))}(size(x)), x) -complexfloat{T<:Real}(x::AbstractArray{T}) = copy!(Array{typeof(complex(float(one(T))))}(size(x)), x) +complexfloat{T<:Complex}(x::AbstractArray{T}) = copy1(typeof(fftwfloat(one(T))), x) +complexfloat{T<:Real}(x::AbstractArray{T}) = copy1(typeof(complex(fftwfloat(one(T)))), x) + +realfloat{T<:Real}(x::AbstractArray{T}) = copy1(typeof(fftwfloat(one(T))), x) + +# copy to a 1-based array, using circular permutation +function copy1{T}(::Type{T}, x) + y = Array{T}(map(length, indices(x))) + Base.circcopy!(y, x) +end + +to1(x::AbstractArray) = _to1(indices(x), x) +_to1(::Tuple{Base.OneTo,Vararg{Base.OneTo}}, x) = x +_to1(::Tuple, x) = copy1(eltype(x), x) # implementations only need to provide plan_X(x, region) # for X in (:fft, :bfft, ...): for f in (:fft, :bfft, :ifft, :fft!, :bfft!, :ifft!, :rfft) pf = Symbol("plan_", f) @eval begin - $f(x::AbstractArray) = $pf(x) * x - $f(x::AbstractArray, region) = $pf(x, region) * x - $pf(x::AbstractArray; kws...) = $pf(x, 1:ndims(x); kws...) + $f(x::AbstractArray) = (y = to1(x); $pf(y) * y) + $f(x::AbstractArray, region) = (y = to1(x); $pf(y, region) * y) + $pf(x::AbstractArray; kws...) = (y = to1(x); $pf(y, 1:ndims(y); kws...)) end end @@ -187,11 +206,11 @@ for f in (:fft, :bfft, :ifft) $pf{T<:Union{Integer,Rational}}(x::AbstractArray{Complex{T}}, region; kws...) = $pf(complexfloat(x), region; kws...) end end -rfft{T<:Union{Integer,Rational}}(x::AbstractArray{T}, region=1:ndims(x)) = rfft(float(x), region) -plan_rfft{T<:Union{Integer,Rational}}(x::AbstractArray{T}, region; kws...) = plan_rfft(float(x), region; kws...) +rfft{T<:Union{Integer,Rational}}(x::AbstractArray{T}, region=1:ndims(x)) = rfft(realfloat(x), region) +plan_rfft(x::AbstractArray, region; kws...) = plan_rfft(realfloat(x), region; kws...) # only require implementation to provide *(::Plan{T}, ::Array{T}) -*{T}(p::Plan{T}, x::AbstractArray) = p * copy!(Array{T}(size(x)), x) +*{T}(p::Plan{T}, x::AbstractArray) = p * copy1(T, x) # Implementations should also implement A_mul_B!(Y, plan, X) so as to support # pre-allocated output arrays. We don't define * in terms of A_mul_B! diff --git a/base/multidimensional.jl b/base/multidimensional.jl index a6e22e481601b..3773db8701964 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -645,6 +645,22 @@ See also `circshift`. end circshift!(dest::AbstractArray, src, shiftamt) = circshift!(dest, src, (shiftamt...,)) +# For each dimension, we copy the first half of src to the second half +# of dest, and the second half of src to the first half of dest. This +# uses a recursive bifurcation strategy so that these splits can be +# encoded by ranges, which means that we need only one call to `mod` +# per dimension rather than one call per index. +# `rdest` and `rsrc` are tuples-of-ranges that grow one dimension at a +# time; when all the dimensions have been filled in, you call `copy!` +# for that block. In other words, in two dimensions schematically we +# have the following call sequence (--> means a call): +# circshift!(dest, src, shiftamt) --> +# _circshift!(dest, src, ("first half of dim1",)) --> +# _circshift!(dest, src, ("first half of dim1", "first half of dim2")) --> copy! +# _circshift!(dest, src, ("first half of dim1", "second half of dim2")) --> copy! +# _circshift!(dest, src, ("second half of dim1",)) --> +# _circshift!(dest, src, ("second half of dim1", "first half of dim2")) --> copy! +# _circshift!(dest, src, ("second half of dim1", "second half of dim2")) --> copy! @inline function _circshift!(dest, rdest, src, rsrc, inds::Tuple{AbstractUnitRange,Vararg{Any}}, shiftamt::Tuple{Integer,Vararg{Any}}) @@ -662,6 +678,68 @@ function _circshift!(dest, rdest, src, rsrc, inds, shiftamt) copy!(dest, CartesianRange(rdest), src, CartesianRange(rsrc)) end +# circcopy! +""" + circcopy!(dest, src) + +Copy `src` to `dest`, indexing each dimension modulo its length. +`src` and `dest` must have the same size, but can be offset in +their indices; any offset results in a (circular) wraparound. If the +arrays have overlapping indices, then on the domain of the overlap +`dest` agrees with `src`. + +```julia +julia> src = reshape(collect(1:16), (4,4)) +4×4 Array{Int64,2}: + 1 5 9 13 + 2 6 10 14 + 3 7 11 15 + 4 8 12 16 + +julia> dest = OffsetArray{Int}((0:3,2:5)) + +julia> circcopy!(dest, src) +OffsetArrays.OffsetArray{Int64,2,Array{Int64,2}} with indices 0:3×2:5: + 8 12 16 4 + 5 9 13 1 + 6 10 14 2 + 7 11 15 3 + +julia> dest[1:3,2:4] == src[1:3,2:4] +true +``` +""" +function circcopy!(dest, src) + dest === src && throw(ArgumentError("dest and src must be separate arrays")) + indssrc, indsdest = indices(src), indices(dest) + if (szsrc = map(length, indssrc)) != (szdest = map(length, indsdest)) + throw(DimensionMismatch("src and dest must have the same sizes (got $szsrc and $szdest)")) + end + shift = map((isrc, idest)->first(isrc)-first(idest), indssrc, indsdest) + all(x->x==0, shift) && return copy!(dest, src) + _circcopy!(dest, (), indsdest, src, (), indssrc) +end + +# This uses the same strategy described above for _circshift! +@inline function _circcopy!(dest, rdest, indsdest::Tuple{AbstractUnitRange,Vararg{Any}}, + src, rsrc, indssrc::Tuple{AbstractUnitRange,Vararg{Any}}) + indd1, inds1 = indsdest[1], indssrc[1] + l = length(indd1) + s = mod(first(inds1)-first(indd1), l) + sdf = first(indd1)+s + rd1, rd2 = first(indd1):sdf-1, sdf:last(indd1) + ssf = last(inds1)-s + rs1, rs2 = first(inds1):ssf, ssf+1:last(inds1) + tindsd, tindss = tail(indsdest), tail(indssrc) + _circcopy!(dest, (rdest..., rd1), tindsd, src, (rsrc..., rs2), tindss) + _circcopy!(dest, (rdest..., rd2), tindsd, src, (rsrc..., rs1), tindss) +end + +# At least one of indsdest, indssrc are empty (and both should be, since we've checked) +function _circcopy!(dest, rdest, indsdest, src, rsrc, indssrc) + copy!(dest, CartesianRange(rdest), src, CartesianRange(rsrc)) +end + ### BitArrays ## getindex diff --git a/test/fft.jl b/test/fft.jl index 6905bc85ffb30..b30e05ff09a5a 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -326,3 +326,11 @@ for x in (randn(10),randn(10,12)) # note: inference doesn't work for plan_fft_ since the # algorithm steps are included in the CTPlan type end + +# issue #17896 +a = rand(5) +@test fft(a) == fft(view(a,:)) == fft(view(a, 1:5)) == fft(view(a, [1:5;])) +@test rfft(a) == rfft(view(a,:)) == rfft(view(a, 1:5)) == rfft(view(a, [1:5;])) +a16 = convert(Vector{Float16}, a) +@test fft(a16) == fft(view(a16,:)) == fft(view(a16, 1:5)) == fft(view(a16, [1:5;])) +@test rfft(a16) == rfft(view(a16,:)) == rfft(view(a16, 1:5)) == rfft(view(a16, [1:5;])) diff --git a/test/offsetarray.jl b/test/offsetarray.jl index a509e17d3a8fb..dac5aad7995ca 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -411,10 +411,41 @@ v = OffsetArray(rand(8), (-2,)) @test rotr90(A) == OffsetArray(rotr90(parent(A)), A.offsets[[2,1]]) @test flipdim(A, 1) == OffsetArray(flipdim(parent(A), 1), A.offsets) @test flipdim(A, 2) == OffsetArray(flipdim(parent(A), 2), A.offsets) -@test circshift(A, (-1,2)) == OffsetArray(circshift(parent(A), (-1,2)), A.offsets) @test A+1 == OffsetArray(parent(A)+1, A.offsets) @test 2*A == OffsetArray(2*parent(A), A.offsets) @test A+A == OffsetArray(parent(A)+parent(A), A.offsets) @test A.*A == OffsetArray(parent(A).*parent(A), A.offsets) + +@test circshift(A, (-1,2)) == OffsetArray(circshift(parent(A), (-1,2)), A.offsets) + +src = reshape(collect(1:16), (4,4)) +dest = OffsetArray(Array{Int}(4,4), (-1,1)) +circcopy!(dest, src) +@test parent(dest) == [8 12 16 4; 5 9 13 1; 6 10 14 2; 7 11 15 3] +@test dest[1:3,2:4] == src[1:3,2:4] + +e = eye(5) +a = [e[:,1], e[:,2], e[:,3], e[:,4], e[:,5]] +a1 = zeros(5) +c = [ones(Complex{Float64}, 5), + exp(-2*pi*im*(0:4)/5), + exp(-4*pi*im*(0:4)/5), + exp(-6*pi*im*(0:4)/5), + exp(-8*pi*im*(0:4)/5)] +for s = -5:5 + for i = 1:5 + thisa = OffsetArray(a[i], (s,)) + thisc = c[mod1(i+s+5,5)] + @test_approx_eq fft(thisa) thisc + @test_approx_eq fft(thisa, 1) thisc + @test_approx_eq ifft(fft(thisa)) circcopy!(a1, thisa) + @test_approx_eq ifft(fft(thisa, 1), 1) circcopy!(a1, thisa) + @test_approx_eq rfft(thisa) thisc[1:3] + @test_approx_eq rfft(thisa, 1) thisc[1:3] + @test_approx_eq irfft(rfft(thisa, 1), 5, 1) a1 + @test_approx_eq irfft(rfft(thisa, 1), 5, 1) a1 + end end + +end # let From 14a5744fcbbcb90e83cb42550c014e2244bd4100 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 10 Aug 2016 21:02:26 -0500 Subject: [PATCH 010/114] Move OffsetArray definition to TestHelpers (cherry picked from commit e95b5e20c4e7d6abfd4345d9dc5a114c6153ffa6) ref #17919 --- test/TestHelpers.jl | 97 ++++++++++++++++++++++++++++++++++++++ test/offsetarray.jl | 110 +++----------------------------------------- 2 files changed, 104 insertions(+), 103 deletions(-) diff --git a/test/TestHelpers.jl b/test/TestHelpers.jl index b57b0d10f77ee..fef9a3089d6aa 100644 --- a/test/TestHelpers.jl +++ b/test/TestHelpers.jl @@ -43,4 +43,101 @@ function with_fake_pty(f) close(master) end +# OffsetArrays (arrays with indexing that doesn't start at 1) + +# This test file is designed to exercise support for generic indexing, +# even though offset arrays aren't implemented in Base. + +module OAs + +using Base: Indices, LinearSlow, LinearFast, tail + +export OffsetArray + +immutable OffsetArray{T,N,AA<:AbstractArray} <: AbstractArray{T,N} + parent::AA + offsets::NTuple{N,Int} +end +typealias OffsetVector{T,AA<:AbstractArray} OffsetArray{T,1,AA} + +OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::NTuple{N,Int}) = OffsetArray{T,N,typeof(A)}(A, offsets) +OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::Vararg{Int,N}) = OffsetArray(A, offsets) + +(::Type{OffsetArray{T,N}}){T,N}(inds::Indices{N}) = OffsetArray{T,N,Array{T,N}}(Array{T,N}(map(length, inds)), map(indsoffset, inds)) +(::Type{OffsetArray{T}}){T,N}(inds::Indices{N}) = OffsetArray{T,N}(inds) + +Base.linearindexing{T<:OffsetArray}(::Type{T}) = Base.linearindexing(parenttype(T)) +parenttype{T,N,AA}(::Type{OffsetArray{T,N,AA}}) = AA +parenttype(A::OffsetArray) = parenttype(typeof(A)) + +Base.parent(A::OffsetArray) = A.parent + +errmsg(A) = error("size not supported for arrays with indices $(indices(A)); see http://docs.julialang.org/en/latest/devdocs/offset-arrays/") +Base.size(A::OffsetArray) = errmsg(A) +Base.size(A::OffsetArray, d) = errmsg(A) +Base.eachindex(::LinearSlow, A::OffsetArray) = CartesianRange(indices(A)) +Base.eachindex(::LinearFast, A::OffsetVector) = indices(A, 1) + +# Implementations of indices and indices1. Since bounds-checking is +# performance-critical and relies on indices, these are usually worth +# optimizing thoroughly. +@inline Base.indices(A::OffsetArray, d) = 1 <= d <= length(A.offsets) ? indices(parent(A))[d] + A.offsets[d] : (1:1) +@inline Base.indices(A::OffsetArray) = _indices(indices(parent(A)), A.offsets) # would rather use ntuple, but see #15276 +@inline _indices(inds, offsets) = (inds[1]+offsets[1], _indices(tail(inds), tail(offsets))...) +_indices(::Tuple{}, ::Tuple{}) = () +Base.indices1{T}(A::OffsetArray{T,0}) = 1:1 # we only need to specialize this one + +function Base.similar(A::OffsetArray, T::Type, dims::Dims) + B = similar(parent(A), T, dims) +end +function Base.similar(A::AbstractArray, T::Type, inds::Tuple{UnitRange,Vararg{UnitRange}}) + B = similar(A, T, map(length, inds)) + OffsetArray(B, map(indsoffset, inds)) +end + +Base.similar(f::Union{Function,DataType}, shape::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(f(map(length, shape)), map(indsoffset, shape)) + +Base.reshape(A::AbstractArray, inds::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(reshape(A, map(length, inds)), map(indsoffset, inds)) + +@inline function Base.getindex{T,N}(A::OffsetArray{T,N}, I::Vararg{Int,N}) + checkbounds(A, I...) + @inbounds ret = parent(A)[offset(A.offsets, I)...] + ret +end +@inline function Base._getindex(::LinearFast, A::OffsetVector, i::Int) + checkbounds(A, i) + @inbounds ret = parent(A)[offset(A.offsets, (i,))[1]] + ret +end +@inline function Base._getindex(::LinearFast, A::OffsetArray, i::Int) + checkbounds(A, i) + @inbounds ret = parent(A)[i] + ret +end +@inline function Base.setindex!{T,N}(A::OffsetArray{T,N}, val, I::Vararg{Int,N}) + checkbounds(A, I...) + @inbounds parent(A)[offset(A.offsets, I)...] = val + val +end +@inline function Base._setindex!(::LinearFast, A::OffsetVector, val, i::Int) + checkbounds(A, i) + @inbounds parent(A)[offset(A.offsets, (i,))[1]] = val + val +end +@inline function Base._setindex!(::LinearFast, A::OffsetArray, val, i::Int) + checkbounds(A, i) + @inbounds parent(A)[i] = val + val +end + +# Computing a shifted index (subtracting the offset) +offset{N}(offsets::NTuple{N,Int}, inds::NTuple{N,Int}) = _offset((), offsets, inds) +_offset(out, ::Tuple{}, ::Tuple{}) = out +@inline _offset(out, offsets, inds) = _offset((out..., inds[1]-offsets[1]), Base.tail(offsets), Base.tail(inds)) + +indsoffset(r::Range) = first(r) - 1 +indsoffset(i::Integer) = 0 + +end + end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index dac5aad7995ca..2136e4dd580e0 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -1,103 +1,7 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -# OffsetArrays (arrays with indexing that doesn't start at 1) - -# This test file is designed to exercise support for generic indexing, -# even though offset arrays aren't implemented in Base. - -module OAs - -using Base: Indices, LinearSlow, LinearFast, tail - -export OffsetArray - -immutable OffsetArray{T,N,AA<:AbstractArray} <: AbstractArray{T,N} - parent::AA - offsets::NTuple{N,Int} -end -typealias OffsetVector{T,AA<:AbstractArray} OffsetArray{T,1,AA} - -OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::NTuple{N,Int}) = OffsetArray{T,N,typeof(A)}(A, offsets) -OffsetArray{T,N}(A::AbstractArray{T,N}, offsets::Vararg{Int,N}) = OffsetArray(A, offsets) - -(::Type{OffsetArray{T,N}}){T,N}(inds::Indices{N}) = OffsetArray{T,N,Array{T,N}}(Array{T,N}(map(length, inds)), map(indsoffset, inds)) -(::Type{OffsetArray{T}}){T,N}(inds::Indices{N}) = OffsetArray{T,N}(inds) - -Base.linearindexing{T<:OffsetArray}(::Type{T}) = Base.linearindexing(parenttype(T)) -parenttype{T,N,AA}(::Type{OffsetArray{T,N,AA}}) = AA -parenttype(A::OffsetArray) = parenttype(typeof(A)) - -Base.parent(A::OffsetArray) = A.parent - -errmsg(A) = error("size not supported for arrays with indices $(indices(A)); see http://docs.julialang.org/en/latest/devdocs/offset-arrays/") -Base.size(A::OffsetArray) = errmsg(A) -Base.size(A::OffsetArray, d) = errmsg(A) -Base.eachindex(::LinearSlow, A::OffsetArray) = CartesianRange(indices(A)) -Base.eachindex(::LinearFast, A::OffsetVector) = indices(A, 1) - -# Implementations of indices and indices1. Since bounds-checking is -# performance-critical and relies on indices, these are usually worth -# optimizing thoroughly. -@inline Base.indices(A::OffsetArray, d) = 1 <= d <= length(A.offsets) ? indices(parent(A))[d] + A.offsets[d] : (1:1) -@inline Base.indices(A::OffsetArray) = _indices(indices(parent(A)), A.offsets) # would rather use ntuple, but see #15276 -@inline _indices(inds, offsets) = (inds[1]+offsets[1], _indices(tail(inds), tail(offsets))...) -_indices(::Tuple{}, ::Tuple{}) = () -Base.indices1{T}(A::OffsetArray{T,0}) = 1:1 # we only need to specialize this one - -function Base.similar(A::OffsetArray, T::Type, dims::Dims) - B = similar(parent(A), T, dims) -end -function Base.similar(A::AbstractArray, T::Type, inds::Tuple{UnitRange,Vararg{UnitRange}}) - B = similar(A, T, map(length, inds)) - OffsetArray(B, map(indsoffset, inds)) -end - -Base.similar(f::Union{Function,DataType}, shape::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(f(map(length, shape)), map(indsoffset, shape)) - -Base.reshape(A::AbstractArray, inds::Tuple{UnitRange,Vararg{UnitRange}}) = OffsetArray(reshape(A, map(length, inds)), map(indsoffset, inds)) - -@inline function Base.getindex{T,N}(A::OffsetArray{T,N}, I::Vararg{Int,N}) - checkbounds(A, I...) - @inbounds ret = parent(A)[offset(A.offsets, I)...] - ret -end -@inline function Base._getindex(::LinearFast, A::OffsetVector, i::Int) - checkbounds(A, i) - @inbounds ret = parent(A)[offset(A.offsets, (i,))[1]] - ret -end -@inline function Base._getindex(::LinearFast, A::OffsetArray, i::Int) - checkbounds(A, i) - @inbounds ret = parent(A)[i] - ret -end -@inline function Base.setindex!{T,N}(A::OffsetArray{T,N}, val, I::Vararg{Int,N}) - checkbounds(A, I...) - @inbounds parent(A)[offset(A.offsets, I)...] = val - val -end -@inline function Base._setindex!(::LinearFast, A::OffsetVector, val, i::Int) - checkbounds(A, i) - @inbounds parent(A)[offset(A.offsets, (i,))[1]] = val - val -end -@inline function Base._setindex!(::LinearFast, A::OffsetArray, val, i::Int) - checkbounds(A, i) - @inbounds parent(A)[i] = val - val -end - -# Computing a shifted index (subtracting the offset) -offset{N}(offsets::NTuple{N,Int}, inds::NTuple{N,Int}) = _offset((), offsets, inds) -_offset(out, ::Tuple{}, ::Tuple{}) = out -@inline _offset(out, offsets, inds) = _offset((out..., inds[1]-offsets[1]), Base.tail(offsets), Base.tail(inds)) - -indsoffset(r::Range) = first(r) - 1 -indsoffset(i::Integer) = 0 - -end - -using OAs +isdefined(:TestHelpers) || include(joinpath(dirname(@__FILE__), "TestHelpers.jl")) +using TestHelpers.OAs let # Basics @@ -219,11 +123,11 @@ cmp_showf(Base.print_matrix, io, OffsetArray(rand(5,5), (10,-9))) # rows&c cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,5), (10,-9))) # columns fit cmp_showf(Base.print_matrix, io, OffsetArray(rand(5,10^3), (10,-9))) # rows fit cmp_showf(Base.print_matrix, io, OffsetArray(rand(10^3,10^3), (10,-9))) # neither fits -targets1 = ["0-dimensional OAs.OffsetArray{Float64,0,Array{Float64,0}}:\n1.0", - "OAs.OffsetArray{Float64,1,Array{Float64,1}} with indices 2:2:\n 1.0", - "OAs.OffsetArray{Float64,2,Array{Float64,2}} with indices 2:2×3:3:\n 1.0", - "OAs.OffsetArray{Float64,3,Array{Float64,3}} with indices 2:2×3:3×4:4:\n[:, :, 4] =\n 1.0", - "OAs.OffsetArray{Float64,4,Array{Float64,4}} with indices 2:2×3:3×4:4×5:5:\n[:, :, 4, 5] =\n 1.0"] +targets1 = ["0-dimensional TestHelpers.OAs.OffsetArray{Float64,0,Array{Float64,0}}:\n1.0", + "TestHelpers.OAs.OffsetArray{Float64,1,Array{Float64,1}} with indices 2:2:\n 1.0", + "TestHelpers.OAs.OffsetArray{Float64,2,Array{Float64,2}} with indices 2:2×3:3:\n 1.0", + "TestHelpers.OAs.OffsetArray{Float64,3,Array{Float64,3}} with indices 2:2×3:3×4:4:\n[:, :, 4] =\n 1.0", + "TestHelpers.OAs.OffsetArray{Float64,4,Array{Float64,4}} with indices 2:2×3:3×4:4×5:5:\n[:, :, 4, 5] =\n 1.0"] targets2 = ["(1.0,1.0)", "([1.0],[1.0])", "(\n[1.0],\n\n[1.0])", From df5286a8b317d7e33b3a2da21d6a3720c4ee56fe Mon Sep 17 00:00:00 2001 From: pabloferz Date: Tue, 9 Aug 2016 21:01:50 +0200 Subject: [PATCH 011/114] Improve inferability of promote_op (cherry picked from commit 6fd91b287f63edc301a2ab219b7a145f5c61930d) ref #17929 --- base/abstractarray.jl | 10 +++++----- base/arraymath.jl | 24 ++++++++++++------------ base/promotion.jl | 23 +++++++++++------------ 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index ef2b9fa100be3..2a292343bc9d0 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1653,11 +1653,11 @@ end # These are needed because map(eltype, As) is not inferrable promote_eltype_op(::Any) = (@_pure_meta; Bottom) -promote_eltype_op(op, A) = (@_pure_meta; _promote_op(op, eltype(A))) -promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; _promote_op(op, T)) -promote_eltype_op{T}(op, ::AbstractArray{T}, A) = (@_pure_meta; _promote_op(op, T, eltype(A))) -promote_eltype_op{T}(op, A, ::AbstractArray{T}) = (@_pure_meta; _promote_op(op, eltype(A), T)) -promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; _promote_op(op, R, S)) +promote_eltype_op(op, A) = (@_pure_meta; promote_op(op, eltype(A))) +promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T)) +promote_eltype_op{T}(op, ::AbstractArray{T}, A) = (@_pure_meta; promote_op(op, T, eltype(A))) +promote_eltype_op{T}(op, A, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, eltype(A), T)) +promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S)) promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_eltype_op(op, promote_eltype_op(op, A, B), C, D...)) ## 1 argument diff --git a/base/arraymath.jl b/base/arraymath.jl index 0de1bd39486f5..171d301fb48e9 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -46,8 +46,8 @@ promote_array_type{S<:Integer}(::typeof(.\), ::Type{S}, ::Type{Bool}, T::Type) = promote_array_type{S<:Integer}(F, ::Type{S}, ::Type{Bool}, T::Type) = T for f in (:+, :-, :div, :mod, :&, :|, :$) - @eval ($f)(A::AbstractArray, B::AbstractArray) = - _elementwise($f, promote_op($f, eltype(A), eltype(B)), A, B) + @eval ($f){R,S}(A::AbstractArray{R}, B::AbstractArray{S}) = + _elementwise($f, promote_op($f, R, S), A, B) end function _elementwise(op, ::Type{Any}, A::AbstractArray, B::AbstractArray) promote_shape(A, B) # check size compatibility @@ -63,21 +63,21 @@ end for f in (:.+, :.-, :.*, :./, :.\, :.^, :.÷, :.%, :.<<, :.>>, :div, :mod, :rem, :&, :|, :$) @eval begin - function ($f)(A::Number, B::AbstractArray) - P = promote_op($f, typeof(A), eltype(B)) - T = promote_array_type($f, typeof(A), eltype(B), P) - T === Any && return [($f)(A, b) for b in B] - F = similar(B, T) + function ($f){T}(A::Number, B::AbstractArray{T}) + R = promote_op($f, typeof(A), T) + S = promote_array_type($f, typeof(A), T, R) + S === Any && return [($f)(A, b) for b in B] + F = similar(B, S) for (iF, iB) in zip(eachindex(F), eachindex(B)) @inbounds F[iF] = ($f)(A, B[iB]) end return F end - function ($f)(A::AbstractArray, B::Number) - P = promote_op($f, eltype(A), typeof(B)) - T = promote_array_type($f, typeof(B), eltype(A), P) - T === Any && return [($f)(a, B) for a in A] - F = similar(A, T) + function ($f){T}(A::AbstractArray{T}, B::Number) + R = promote_op($f, T, typeof(B)) + S = promote_array_type($f, typeof(B), T, R) + S === Any && return [($f)(a, B) for a in A] + F = similar(A, S) for (iF, iA) in zip(eachindex(F), eachindex(A)) @inbounds F[iF] = ($f)(A[iA], B) end diff --git a/base/promotion.jl b/base/promotion.jl index 449dae5add81a..4952126dc8cd8 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -220,19 +220,17 @@ minmax(x::Real, y::Real) = minmax(promote(x, y)...) # "Promotion" that takes a function into account. These are meant to be # used mainly by broadcast methods, so it is advised against overriding them if isdefined(Core, :Inference) - function _promote_op(op, T::Type) + function _promote_op(op, T::ANY) G = Tuple{Generator{Tuple{T},typeof(op)}} - R = Core.Inference.return_type(first, G) - return isleaftype(R) ? R : Any + return Core.Inference.return_type(first, G) end - function _promote_op(op, R::Type, S::Type) + function _promote_op(op, R::ANY, S::ANY) F = typeof(a -> op(a...)) G = Tuple{Generator{Zip2{Tuple{R},Tuple{S}},F}} - T = Core.Inference.return_type(first, G) - return isleaftype(T) ? T : Any + return Core.Inference.return_type(first, G) end else - _promote_op(::Any...) = (@_pure_meta; Any) + _promote_op(::ANY...) = (@_pure_meta; Any) end _default_type(T::Type) = (@_pure_meta; T) @@ -240,14 +238,15 @@ promote_op(::Any...) = (@_pure_meta; Any) promote_op(T::Type, ::Any) = (@_pure_meta; T) promote_op(T::Type, ::Type) = (@_pure_meta; T) # To handle ambiguities # Promotion that tries to preserve non-concrete types -function promote_op(f, S::Type) +function promote_op{S}(f, ::Type{S}) T = _promote_op(f, _default_type(S)) - return isleaftype(S) ? T : typejoin(S, T) + isleaftype(S) && return isleaftype(T) ? T : Any + return typejoin(S, T) end -function promote_op(f, R::Type, S::Type) +function promote_op{R,S}(f, ::Type{R}, ::Type{S}) T = _promote_op(f, _default_type(R), _default_type(S)) - isleaftype(R) && return isleaftype(S) ? T : typejoin(S, T) - return isleaftype(S) ? typejoin(R, T) : typejoin(R, S, T) + isleaftype(R) && isleaftype(S) && return isleaftype(T) ? T : Any + return typejoin(R, S, T) end ## catch-alls to prevent infinite recursion when definitions are missing ## From 6dd9e814a0f5d5f2d40ecf7c54b93743e2ce59cc Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 11 Aug 2016 08:30:47 -0500 Subject: [PATCH 012/114] Fix StackOverflowError in zip_iteratorsize (cherry picked from commit 1e7f92c5163e0a638c537ac3babb2e5b33dc5ea8) ref #17966 --- base/iterator.jl | 1 + test/functional.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/base/iterator.jl b/base/iterator.jl index 3d74fd5da57db..09dca34b880e4 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -61,6 +61,7 @@ zip_iteratorsize(a, b) = and_iteratorsize(a,b) # as `and_iteratorsize` but inher zip_iteratorsize(::HasLength, ::IsInfinite) = HasLength() zip_iteratorsize(::HasShape, ::IsInfinite) = HasLength() zip_iteratorsize(a::IsInfinite, b) = zip_iteratorsize(b,a) +zip_iteratorsize(a::IsInfinite, b::IsInfinite) = IsInfinite() immutable Zip1{I} <: AbstractZipIterator diff --git a/test/functional.jl b/test/functional.jl index b495ddb715847..5ed8f3ce51db8 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -185,6 +185,7 @@ end @test Base.iteratorsize(repeated(0, 5)) == Base.HasLength() @test Base.iteratoreltype(repeated(0)) == Base.HasEltype() @test Base.iteratoreltype(repeated(0, 5)) == Base.HasEltype() +@test Base.iteratorsize(zip(repeated(0), repeated(0))) == Base.IsInfinite() # product From dcea839384bbcb5484736aa5134b6d604cab1cad Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 11 Aug 2016 10:49:53 -0400 Subject: [PATCH 013/114] slightly improve error message for unsupported kw arguments also fix some code formatting (cherry picked from commit 28d8b80f6a59614fc005accf50f15d57d60bb01f) ref #17973 --- base/replutil.jl | 3 ++- src/julia-syntax.scm | 5 +++-- test/keywordargs.jl | 2 +- test/replutil.jl | 24 ++++++++++++------------ 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/base/replutil.jl b/base/replutil.jl index 3c26308f06b64..fbd14f79b4608 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -533,7 +533,8 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs::Vector=Any[]) end if !isempty(unexpected) Base.with_output_color(:red, buf) do buf - print(buf, " got an unsupported keyword argument \"", join(unexpected, "\", \""), "\"") + plur = length(unexpected) > 1 ? "s" : "" + print(buf, " got unsupported keyword argument$plur \"", join(unexpected, "\", \""), "\"") end end end diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index f456a68bf1ea4..2e483095e1952 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -522,8 +522,9 @@ ,else))) (if (null? restkw) ;; if no rest kw, give error for unrecognized - `(call (top kwerr) ,kw ,@(map arg-name pargl),@(if (null? vararg) '() - (list `(... ,(arg-name (car vararg)))))) + `(call (top kwerr) ,kw ,@(map arg-name pargl) + ,@(if (null? vararg) '() + (list `(... ,(arg-name (car vararg)))))) ;; otherwise add to rest keywords `(ccall 'jl_array_ptr_1d_push Void (tuple Any Any) ,rkw (tuple ,elt diff --git a/test/keywordargs.jl b/test/keywordargs.jl index 7a582b73645f1..8030e3dce0ee1 100644 --- a/test/keywordargs.jl +++ b/test/keywordargs.jl @@ -9,7 +9,7 @@ kwf1(ones; tens=0, hundreds=0) = ones + 10*tens + 100*hundreds @test kwf1(3, tens=7, hundreds=2) == 273 @test_throws MethodError kwf1() # no method, too few args -@test_throws MethodError kwf1(1, z=0) # unsupported keyword +@test_throws MethodError kwf1(1, z=0) # unsupported keyword @test_throws MethodError kwf1(1, 2) # no method, too many positional args # keyword args plus varargs diff --git a/test/replutil.jl b/test/replutil.jl index 4ad026f3a8f80..54e1f2a7aaf2c 100644 --- a/test/replutil.jl +++ b/test/replutil.jl @@ -127,19 +127,19 @@ showerror(buf, m_error) error_out3 = takebuf_string(buf) if Base.have_color - @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") + @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1)\e[1m\e[31m got unsupported keyword argument \"y\"\e[0m") @test contains(error_out, "method_c6(\e[1m\e[31m::Any\e[0m; y)$cfile$(c6line + 2)") - @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") - @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2)\e[1m\e[31m got an unsupported keyword argument \"y\"\e[0m") + @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2)\e[1m\e[31m got unsupported keyword argument \"x\"\e[0m") + @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2)\e[1m\e[31m got unsupported keyword argument \"y\"\e[0m") @test contains(error_out2, "method_c6_in_module(\e[1m\e[31m::Any\e[0m; y)$cfile$(c6mline + 3)") - @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3)\e[1m\e[31m got an unsupported keyword argument \"x\"\e[0m") + @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3)\e[1m\e[31m got unsupported keyword argument \"x\"\e[0m") else - @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1) got an unsupported keyword argument \"y\"") + @test contains(error_out, "method_c6(; x)$cfile$(c6line + 1) got unsupported keyword argument \"y\"") @test contains(error_out, "method_c6(!Matched::Any; y)$cfile$(c6line + 2)") - @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2) got an unsupported keyword argument \"x\"") - @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2) got an unsupported keyword argument \"y\"") + @test contains(error_out1, "method_c6(::Any; y)$cfile$(c6line + 2) got unsupported keyword argument \"x\"") + @test contains(error_out2, "method_c6_in_module(; x)$cfile$(c6mline + 2) got unsupported keyword argument \"y\"") @test contains(error_out2, "method_c6_in_module(!Matched::Any; y)$cfile$(c6mline + 3)") - @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3) got an unsupported keyword argument \"x\"") + @test contains(error_out3, "method_c6_in_module(::Any; y)$cfile$(c6mline + 3) got unsupported keyword argument \"x\"") end c7line = @__LINE__ + 1 @@ -150,16 +150,16 @@ test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c7(::Any, ::Any; c8line = @__LINE__ + 1 method_c8(a, b; y=1, w=1) = a Base.show_method_candidates(buf, MethodError(method_c8, (1, 1)), [(:x, 1), (:y, 2), (:z, 1), (:w, 1)]) -test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line\e[1m\e[31m got an unsupported keyword argument \"x\", \"z\"\e[0m\e[0m", - "\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line got an unsupported keyword argument \"x\", \"z\"") +test_have_color(buf, "\e[0m\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line\e[1m\e[31m got unsupported keyword arguments \"x\", \"z\"\e[0m\e[0m", + "\nClosest candidates are:\n method_c8(::Any, ::Any; y, w)$cfile$c8line got unsupported keyword arguments \"x\", \"z\"") ac15639line = @__LINE__ addConstraint_15639(c::Int32) = c addConstraint_15639(c::Int64; uncset=nothing) = addConstraint_15639(Int32(c), uncset=uncset) Base.show_method_candidates(buf, MethodError(addConstraint_15639, (Int32(1),)), [(:uncset, nothing)]) -test_have_color(buf, "\e[0m\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1)\e[1m\e[31m got an unsupported keyword argument \"uncset\"\e[0m\n addConstraint_15639(\e[1m\e[31m::Int64\e[0m; uncset)$cfile$(ac15639line + 2)\e[0m", - "\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1) got an unsupported keyword argument \"uncset\"\n addConstraint_15639(!Matched::Int64; uncset)$cfile$(ac15639line + 2)") +test_have_color(buf, "\e[0m\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1)\e[1m\e[31m got unsupported keyword argument \"uncset\"\e[0m\n addConstraint_15639(\e[1m\e[31m::Int64\e[0m; uncset)$cfile$(ac15639line + 2)\e[0m", + "\nClosest candidates are:\n addConstraint_15639(::Int32)$cfile$(ac15639line + 1) got unsupported keyword argument \"uncset\"\n addConstraint_15639(!Matched::Int64; uncset)$cfile$(ac15639line + 2)") macro except_str(expr, err_type) return quote From 721c6b7125b2aa72cf468678a352d54f14d6b088 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 10 Aug 2016 18:16:20 -0400 Subject: [PATCH 014/114] improve correctness of fieldtype_tfunc Vararg is only exact if in covariant position also make getfield_tfunc monotonic for the case where the type has one field, to avoid the same bug fix #16530 (cherry picked from commit aeaff55cd606583a06b8c7c2a174cbbbb2577728) ref #17953 --- base/inference.jl | 21 ++++++++++++++++++--- src/jltypes.c | 6 +++--- test/inference.jl | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 4022da0726a60..92874f6f7b813 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -408,7 +408,7 @@ function limit_type_depth(t::ANY, d::Int, cov::Bool, vars) else return t end - if inexact && !isvarargtype(R) + if inexact && (!cov || !isvarargtype(R)) R = TypeVar(:_,R) push!(vars, R) end @@ -467,12 +467,15 @@ function getfield_tfunc(s0::ANY, name) end end snames = s.name.names - for i=1:length(snames) + for i = 1:length(snames) if is(snames[i],fld) R = s.types[i] if isempty(s.parameters) return R, true else + # conservatively limit the type depth here, + # since the UnionAll type bound is otherwise incorrect + # in the current type system typ = limit_type_depth(R, 0, true, filter!(x->isa(x,TypeVar), Any[s.parameters...])) return typ, isleaftype(s) && typeseq(typ, R) @@ -493,8 +496,20 @@ function getfield_tfunc(s0::ANY, name) return Bottom, true end return s.types[i], false + elseif isempty(s.types) + return Bottom, true + elseif length(s.types) == 1 && isempty(s.parameters) + return s.types[1], true else - return reduce(tmerge, Bottom, map(unwrapva,s.types)) #=Union{s.types...}=#, false + R = reduce(tmerge, Bottom, map(unwrapva,s.types)) #=Union{s.types...}=# + # do the same limiting as the known-symbol case to preserve type-monotonicity + if isempty(s.parameters) + return R, typeseq(R, s.types[1]) + else + typ = limit_type_depth(R, 0, true, + filter!(x->isa(x,TypeVar), Any[s.parameters...])) + return typ, isleaftype(s) && typeseq(typ, R) + end end end add_tfunc(getfield, 2, 2, (s,name)->getfield_tfunc(s,name)[1]) diff --git a/src/jltypes.c b/src/jltypes.c index 74f54e98eb768..c6eedef9f8b37 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -83,7 +83,7 @@ static int jl_has_typevars__(jl_value_t *v, int incl_wildcard, jl_value_t **p, s jl_has_typevars__(((jl_tvar_t*)v)->lb, incl_wildcard, p, np)) return 1; if (p != NULL) { - for(i=0; i < np; i++) { + for (i = 0; i < np; i++) { if (v == p[i]) return 1; } @@ -102,7 +102,7 @@ static int jl_has_typevars__(jl_value_t *v, int incl_wildcard, jl_value_t **p, s } else if (jl_is_datatype(v)) { if (is_unspec((jl_datatype_t*)v)) - return 0; + return 0; // TODO: fix expect in this case if (p == NULL) { if (incl_wildcard) expect = ((jl_datatype_t*)v)->haswildcard; @@ -118,7 +118,7 @@ static int jl_has_typevars__(jl_value_t *v, int incl_wildcard, jl_value_t **p, s return 0; } size_t l = jl_svec_len(t); - for(i=0; i < l; i++) { + for (i = 0; i < l; i++) { jl_value_t *elt = jl_svecref(t, i); if (elt != v) { if (jl_has_typevars__(elt, incl_wildcard, p, np)) { diff --git a/test/inference.jl b/test/inference.jl index 2615e9cc4847a..1d99d6539bc14 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -282,3 +282,27 @@ let I = Integer[] push!(I, 1) @test I == Any[1] end + +# issue #16530 +type Foo16530a{dim} + c::Vector{NTuple{dim, Float64}} + d::Vector +end +type Foo16530b{dim} + c::Vector{NTuple{dim, Float64}} +end +f16530a() = fieldtype(Foo16530a, :c) +f16530a(c) = fieldtype(Foo16530a, c) +f16530b() = fieldtype(Foo16530b, :c) +f16530b(c) = fieldtype(Foo16530b, c) + +let T = Array{Tuple{Vararg{Float64,TypeVar(:dim)}},1}, + TTlim = Type{TypeVar(:_,Array{TypeVar(:_,Tuple),1})} + + @test f16530a() == T + @test f16530a(:c) == T + @test Base.return_types(f16530a, ()) == Any[TTlim] + @test Base.return_types(f16530b, ()) == Any[TTlim] + @test Base.return_types(f16530b, (Symbol,)) == Any[TTlim] +end +@test f16530a(:d) == Vector From 0c4397da2c519324d15a7194b5799fb18bc84814 Mon Sep 17 00:00:00 2001 From: Art Date: Thu, 11 Aug 2016 22:08:02 -0400 Subject: [PATCH 015/114] disable openssl in curl build when build agains mbedtls [fix #17910] (#17955) (cherry picked from commit 18160d267d371e80ffeaeb407652a2626e6c7687) --- deps/curl.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/curl.mk b/deps/curl.mk index 12295cde6ad08..0e228d956feaa 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -16,7 +16,7 @@ $(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VE $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure mkdir -p $(dir $@) cd $(dir $@) && \ - $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --with-mbedtls=$(build_shlibdir)/.. CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --without-ssl --with-mbedtls=$(build_prefix) CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" touch -c $@ $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) From a062d65659bb696fedab29306a9f07b14fff8ae8 Mon Sep 17 00:00:00 2001 From: xorJane Date: Thu, 11 Aug 2016 19:09:00 -0700 Subject: [PATCH 016/114] Move documentation inline for error types declared in base/base.jl. (#17959) (cherry picked from commit 87e00a144c77232c5d399e536103fd76a70786a5) --- base/base.jl | 59 ++++++++++++++++++++++++++++++- base/docs/helpdb/Base.jl | 75 ---------------------------------------- 2 files changed, 58 insertions(+), 76 deletions(-) diff --git a/base/base.jl b/base/base.jl index 910db8d8a6fec..9fa8d77cb4a15 100644 --- a/base/base.jl +++ b/base/base.jl @@ -1,5 +1,10 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +""" + SystemError(prefix::AbstractString, [errno::Int32]) + +A system call failed with an error code (in the `errno` global variable). +""" type SystemError <: Exception prefix::AbstractString errnum::Int32 @@ -9,10 +14,22 @@ type SystemError <: Exception SystemError(p::AbstractString) = new(p, Libc.errno()) end +""" + ParseError(msg) + +The expression passed to the `parse` function could not be interpreted as a valid Julia +expression. +""" type ParseError <: Exception msg::AbstractString end +""" + ArgumentError(msg) + +The parameters to a function call do not match a valid signature. Argument `msg` is a +descriptive error string. +""" type ArgumentError <: Exception msg::AbstractString end @@ -21,25 +38,53 @@ end # var::Symbol #end +""" + KeyError(key) + +An indexing operation into an `Associative` (`Dict`) or `Set` like object tried to access or +delete a non-existent element. +""" type KeyError <: Exception key end +""" + MethodError(f, args) + +A method with the required type signature does not exist in the given generic function. +Alternatively, there is no unique most-specific method. +""" type MethodError <: Exception f args end +""" + EOFError() + +No more data was available to read from a file or stream. +""" type EOFError <: Exception end +""" + DimensionMismatch([msg]) + +The objects called do not have matching dimensionality. Optional argument `msg` is a +descriptive error string. +""" type DimensionMismatch <: Exception msg::AbstractString end DimensionMismatch() = DimensionMismatch("") +""" + AssertionError([msg]) + +The asserted condition did not evaluate to `true`. +Optional argument `msg` is a descriptive error string. +""" type AssertionError <: Exception msg::AbstractString - AssertionError() = new("") AssertionError(msg) = new(msg) end @@ -48,12 +93,24 @@ end #Subtypes should put the exception in an 'error' field abstract WrappedException <: Exception +""" + LoadError(file::AbstractString, line::Int, error) + +An error occurred while `include`ing, `require`ing, or `using` a file. The error specifics +should be available in the `.error` field. +""" type LoadError <: WrappedException file::AbstractString line::Int error end +""" + InitError(mod::Symbol, error) + +An error occurred when running a module's `__init__` function. The actual error thrown is +available in the `.error` field. +""" type InitError <: WrappedException mod::Symbol error diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 3a669992c71f3..6d5c6c2f24a2f 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2145,13 +2145,6 @@ Remove each element of `iterable` from set `s` in-place. """ setdiff! -""" - EOFError() - -No more data was available to read from a file or stream. -""" -EOFError - """ isascii(c::Union{Char,AbstractString}) -> Bool @@ -2968,12 +2961,6 @@ handle properly. """ OutOfMemoryError -""" - SystemError(prefix::AbstractString, [errno::Int32]) - -A system call failed with an error code (in the `errno` global variable). -""" -SystemError """ binomial(n,k) @@ -3012,14 +2999,6 @@ detailed system information is shown as well. """ versioninfo -""" - DimensionMismatch([msg]) - -The objects called do not have matching dimensionality. Optional argument `msg` is a -descriptive error string. -""" -DimensionMismatch - """ sort!(v, [alg=,] [by=,] [lt=,] [rev=false]) @@ -3590,13 +3569,6 @@ cannot be used with empty collections (see `reduce(op, itr)`). """ foldr(op, itr) -""" - ParseError(msg) - -The expression passed to the `parse` function could not be interpreted as a valid Julia expression. -""" -ParseError - """ delete!(collection, key) @@ -4449,22 +4421,6 @@ Get the file name part of a path. """ basename -""" - ArgumentError(msg) - -The parameters to a function call do not match a valid signature. Argument `msg` is a -descriptive error string. -""" -ArgumentError - -""" - KeyError(key) - -An indexing operation into an `Associative` (`Dict`) or `Set` like object tried to access or -delete a non-existent element. -""" -KeyError - """ isdiag(A) -> Bool @@ -5321,13 +5277,6 @@ the array length. If the array length is excessive, the excess portion is filled """ digits! -""" - MethodError(f, args) - -A method with the required type signature does not exist in the given generic function. Alternatively, there is no unique most-specific method. -""" -MethodError - """ cat(dims, A...) @@ -6620,22 +6569,6 @@ Compute the phase angle in radians of a complex number `z`. """ angle -""" - LoadError(file::AbstractString, line::Int, error) - -An error occurred while `include`ing, `require`ing, or `using` a file. The error specifics -should be available in the `.error` field. -""" -LoadError - -""" - InitError(mod::Symbol, error) - -An error occurred when running a module's `__init__` function. The actual error thrown is -available in the `.error` field. -""" -InitError - """ copy!(dest, src) @@ -7512,14 +7445,6 @@ Integer division was attempted with a denominator value of 0. """ DivideError -""" - AssertionError([msg]) - -The asserted condition did not evaluate to `true`. -Optional argument `msg` is a descriptive error string. -""" -AssertionError - """ Ac_ldiv_Bc(A, B) From 5cea4568263fcc7937b4dcfb1e954446730de57c Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Thu, 11 Aug 2016 17:34:46 +0200 Subject: [PATCH 017/114] Remove two cross-references * Replace a malformed link in `write` docstring with a literal `write` rather than a real cross-reference, since that would just cross-reference itself. Also move `write` docstring inline. * Remove cross-reference for `<<` from inside a literal since text inside a literal can't contain a link. A cross-reference to `<<` already appears at the end of the docstring. (cherry picked from commit 2301c272135d69a7ebd7d0bf34eeb3bda07aec9d) ref #17969 --- base/docs/helpdb/Base.jl | 15 --------------- base/io.jl | 14 ++++++++++++++ base/operators.jl | 2 +- doc/stdlib/io-network.rst | 2 +- doc/stdlib/math.rst | 2 +- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 6d5c6c2f24a2f..9c6b91207fa98 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2905,21 +2905,6 @@ Squared absolute value of `x`. """ abs2 -""" - write(stream::IO, x) - write(filename::AbstractString, x) - -Write the canonical binary representation of a value to the given I/O stream or file. -Returns the number of bytes written into the stream. - -You can write multiple values with the same :func:`write` call. i.e. the following are -equivalent: - - write(stream, x, y...) - write(stream, x) + write(stream, y...) -""" -write - """ sizehint!(s, n) diff --git a/base/io.jl b/base/io.jl index 29aa202314678..9d65ae5e5e1d4 100644 --- a/base/io.jl +++ b/base/io.jl @@ -23,6 +23,20 @@ function iswritable end function copy end function eof end +""" + write(stream::IO, x) + write(filename::AbstractString, x) + +Write the canonical binary representation of a value to the given I/O stream or file. +Returns the number of bytes written into the stream. + +You can write multiple values with the same `write` call. i.e. the following are equivalent: + + write(stream, x, y...) + write(stream, x) + write(stream, y...) +""" +function write end + read(s::IO, ::Type{UInt8}) = error(typeof(s)," does not support byte I/O") write(s::IO, x::UInt8) = error(typeof(s)," does not support byte I/O") diff --git a/base/operators.jl b/base/operators.jl index 31faf73b1d77f..be5ccf5c5354f 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -230,7 +230,7 @@ end Unsigned right bit shift operator, `x >>> n`. For `n >= 0`, the result is `x` shifted right by `n` bits, where `n >= 0`, filling with `0`s. For `n < 0`, this -is equivalent to `x [<<](:func:`<<`) -n`]. +is equivalent to `x << -n`. For `Unsigned` integer types, this is equivalent to [`>>`](:func:`>>`). For `Signed` integer types, this is equivalent to `signed(unsigned(x) >> n)`. diff --git a/doc/stdlib/io-network.rst b/doc/stdlib/io-network.rst index 4e747354dda2a..67a1b123c57b6 100644 --- a/doc/stdlib/io-network.rst +++ b/doc/stdlib/io-network.rst @@ -134,7 +134,7 @@ General I/O Write the canonical binary representation of a value to the given I/O stream or file. Returns the number of bytes written into the stream. - You can write multiple values with the same :func:``write`` call. i.e. the following are equivalent: + You can write multiple values with the same ``write`` call. i.e. the following are equivalent: .. code-block:: julia diff --git a/doc/stdlib/math.rst b/doc/stdlib/math.rst index cf4a4ad72312d..206f9e38733f7 100644 --- a/doc/stdlib/math.rst +++ b/doc/stdlib/math.rst @@ -284,7 +284,7 @@ Mathematical Operators .. Docstring generated from Julia source - Unsigned right bit shift operator, ``x >>> n``\ . For ``n >= 0``\ , the result is ``x`` shifted right by ``n`` bits, where ``n >= 0``\ , filling with ``0``\ s. For ``n < 0``\ , this is equivalent to ``x [<<](:func:``\ <<``) -n``\ ]. + Unsigned right bit shift operator, ``x >>> n``\ . For ``n >= 0``\ , the result is ``x`` shifted right by ``n`` bits, where ``n >= 0``\ , filling with ``0``\ s. For ``n < 0``\ , this is equivalent to ``x << -n``\ . For ``Unsigned`` integer types, this is equivalent to :func:`>>`\ . For ``Signed`` integer types, this is equivalent to ``signed(unsigned(x) >> n)``\ . From b68e76043846b8612194a46adfc82d2645d5927f Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 11 Aug 2016 16:32:08 -0500 Subject: [PATCH 018/114] Generalize getindex for AbstractUnitRanges It turns out we were returning a Vector{Int} for operations that should return a Range (cherry picked from commit 3bb78151c5e3e17a267e0a0c726d011101e3c817) ref #17980 --- base/range.jl | 5 +++-- test/ranges.jl | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/range.jl b/base/range.jl index caca080802553..00b2b217c1f6a 100644 --- a/base/range.jl +++ b/base/range.jl @@ -499,10 +499,11 @@ end getindex(r::Range, ::Colon) = copy(r) -function getindex{T<:Integer}(r::UnitRange, s::AbstractUnitRange{T}) +function getindex{T<:Integer}(r::AbstractUnitRange, s::AbstractUnitRange{T}) @_inline_meta @boundscheck checkbounds(r, s) - st = oftype(r.start, r.start + first(s)-1) + f = first(r) + st = oftype(f, f + first(s)-1) range(st, length(s)) end diff --git a/test/ranges.jl b/test/ranges.jl index c92ebeb385bbf..b22db9b77a1b4 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -755,6 +755,7 @@ r = Base.OneTo(3) @test minimum(r) == 1 @test maximum(r) == 3 @test r[2] == 2 +@test r[2:3] === 2:3 @test_throws BoundsError r[4] @test_throws BoundsError r[0] @test r+1 === 2:4 From e8db99420f258c9bc1c550b6320ee43f036e419a Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 13 Aug 2016 08:00:04 -0500 Subject: [PATCH 019/114] Silence unneeded output in backtrace test (cherry picked from commit fa367106bea2308e4cd878efd5a51b4ee896d1dd) ref #18013 --- test/backtrace.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/backtrace.jl b/test/backtrace.jl index bdc93b000e0d9..5a73200d79d07 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -121,7 +121,8 @@ end @test_broken hasbt2 function btmacro() - @time backtrace() + ret = @timed backtrace() + ret[1] end lkup = map(StackTraces.lookup, btmacro()) hasme = hasbtmacro = false From 6def1e690b1433273a881af5a8706152f9a0d136 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 13 Aug 2016 12:10:46 -0400 Subject: [PATCH 020/114] stop curl from rebuilding incessantly (cherry picked from commit e4b5233568d6b98b17dffbdc953f2878d2110b3e) fixes #17938 --- deps/curl.mk | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deps/curl.mk b/deps/curl.mk index 0e228d956feaa..b8bf9ccbd8fec 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -1,6 +1,6 @@ ## CURL ## -CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/.libs/libcurl.$(SHLIB_EXT) +CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/lib/.libs/libcurl.$(SHLIB_EXT) CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT) ifneq ($(OS),WINNT) @@ -9,18 +9,22 @@ endif $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2 + $(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET) $(JLCHECKSUM) $< cd $(dir $<) && $(TAR) jxf $(notdir $<) touch -c $@ + $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --without-ssl --with-mbedtls=$(build_prefix) CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" touch -c $@ + $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status $(MAKE) -C $(dir $<) $(LIBTOOL_CCLD) touch -c $@ + $(BUILDDIR)/curl-$(CURL_VER)/checked: $(CURL_SRC_TARGET) ifeq ($(OS),$(BUILD_OS)) ifneq ($(OS),WINNT) @@ -28,6 +32,7 @@ ifneq ($(OS),WINNT) endif endif echo 1 > $@ + $(CURL_OBJ_TARGET): $(CURL_SRC_TARGET) $(call make-install,curl-$(CURL_VER),$(LIBTOOL_CCLD)) $(INSTALL_NAME_CMD)libcurl.$(SHLIB_EXT) $@ From 6538aad0b82735cf8435fbfdf7a8bd5131c7db83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kenta=20Sato=20=28=E4=BD=90=E8=97=A4=20=E5=BB=BA=E5=A4=AA?= =?UTF-8?q?=29?= Date: Sun, 14 Aug 2016 12:30:58 +0900 Subject: [PATCH 021/114] fix a typo in Base.split (#18008) * fix a typo in Base.split * fixup! fix a typo in Base.split (cherry picked from commit 75e88af986d5b8fb7700bf702b41f8a2d2d5b1f4) --- base/docs/helpdb/Base.jl | 2 +- doc/stdlib/strings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index 9c6b91207fa98..d510c91a7b7b2 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -1226,7 +1226,7 @@ Return an array of substrings by splitting the given string on occurrences of th character delimiters, which may be specified in any of the formats allowed by `search`'s second argument (i.e. a single character, collection of characters, string, or regular expression). If `chars` is omitted, it defaults to the set of all space characters, and -`keep` is taken to be `false`. The two keyword arguments are optional: they are are a +`keep` is taken to be `false`. The two keyword arguments are optional: they are a maximum size for the result and a flag determining whether empty fields should be kept in the result. """ diff --git a/doc/stdlib/strings.rst b/doc/stdlib/strings.rst index ff929e55114c2..bfb8999981544 100644 --- a/doc/stdlib/strings.rst +++ b/doc/stdlib/strings.rst @@ -259,7 +259,7 @@ .. Docstring generated from Julia source - Return an array of substrings by splitting the given string on occurrences of the given character delimiters, which may be specified in any of the formats allowed by ``search``\ 's second argument (i.e. a single character, collection of characters, string, or regular expression). If ``chars`` is omitted, it defaults to the set of all space characters, and ``keep`` is taken to be ``false``\ . The two keyword arguments are optional: they are are a maximum size for the result and a flag determining whether empty fields should be kept in the result. + Return an array of substrings by splitting the given string on occurrences of the given character delimiters, which may be specified in any of the formats allowed by ``search``\ 's second argument (i.e. a single character, collection of characters, string, or regular expression). If ``chars`` is omitted, it defaults to the set of all space characters, and ``keep`` is taken to be ``false``\ . The two keyword arguments are optional: they are a maximum size for the result and a flag determining whether empty fields should be kept in the result. .. function:: rsplit(string, [chars]; limit=0, keep=true) From ede72312f4a052049f520570c4ec6cee6aa48389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Sat, 13 Aug 2016 13:17:43 +0200 Subject: [PATCH 022/114] Move env.jl test to separate test file (cherry picked from commit 30afe08b5d4af597588551e132c67a51974557a4) ref #18012 --- test/choosetests.jl | 2 +- test/env.jl | 32 ++++++++++++++++++++++++++++++++ test/sysinfo.jl | 31 ------------------------------- 3 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 test/env.jl diff --git a/test/choosetests.jl b/test/choosetests.jl index d02ed1dcf3cf0..f15e61a139d6c 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -26,7 +26,7 @@ function choosetests(choices = []) "priorityqueue", "file", "read", "mmap", "version", "resolve", "pollfd", "mpfr", "broadcast", "complex", "socket", "floatapprox", "datafmt", "reflection", "regex", "float16", - "combinatorics", "sysinfo", "rounding", "ranges", "mod2pi", + "combinatorics", "sysinfo", "env", "rounding", "ranges", "mod2pi", "euler", "show", "lineedit", "replcompletions", "repl", "replutil", "sets", "test", "goto", "llvmcall", "grisu", "nullable", "meta", "stacktraces", "profile", "libgit2", "docs", diff --git a/test/env.jl b/test/env.jl new file mode 100644 index 0000000000000..592c378cf65ee --- /dev/null +++ b/test/env.jl @@ -0,0 +1,32 @@ +# This file is a part of Julia. License is MIT: http://julialang.org/license + +@test !("f=a=k=e=n=a=m=e" ∈ keys(ENV)) + +# issue #10994 +@test_throws ArgumentError ENV["bad\0name"] = "ok" +@test_throws ArgumentError ENV["okname"] = "bad\0val" +@test_throws ArgumentError Sys.set_process_title("bad\0title") + +withenv("bad"=>"dog") do + @test_throws ArgumentError ENV["bad\0cat"] +end + +# issue #11170 +withenv("TEST"=>"nonempty") do + @test ENV["TEST"] == "nonempty" +end +withenv("TEST"=>"") do + @test ENV["TEST"] == "" +end + +let c = collect(ENV) + @test isa(c, Vector) + @test length(ENV) == length(c) + @test isempty(ENV) || first(ENV) in c +end + +# test for non-existent keys +key = randstring(25) +@test !haskey(ENV,key) +@test_throws KeyError ENV[key] +@test get(ENV,key,"default") == "default" diff --git a/test/sysinfo.jl b/test/sysinfo.jl index 645c34efce765..261935ac715f1 100644 --- a/test/sysinfo.jl +++ b/test/sysinfo.jl @@ -6,34 +6,3 @@ sprint(Base.Sys.cpu_summary) @test Base.Sys.uptime() > 0 Base.Sys.loadavg() - -@test !("f=a=k=e=n=a=m=e" ∈ keys(ENV)) - -# issue #10994 -@test_throws ArgumentError ENV["bad\0name"] = "ok" -@test_throws ArgumentError ENV["okname"] = "bad\0val" -@test_throws ArgumentError Sys.set_process_title("bad\0title") - -withenv("bad"=>"dog") do - @test_throws ArgumentError ENV["bad\0cat"] -end - -# issue #11170 -withenv("TEST"=>"nonempty") do - @test ENV["TEST"] == "nonempty" -end -withenv("TEST"=>"") do - @test ENV["TEST"] == "" -end - -let c = collect(ENV) - @test isa(c, Vector) - @test length(ENV) == length(c) - @test isempty(ENV) || first(ENV) in c -end - -# test for non-existent keys -key = randstring(25) -@test !haskey(ENV,key) -@test_throws KeyError ENV[key] -@test get(ENV,key,"default") == "default" From e1fab8366d026b533d0c3fb2af07555290f4bb4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Sat, 13 Aug 2016 13:20:09 +0200 Subject: [PATCH 023/114] Fix #17956 and add test. The bug was introduced in 103db50aa6c67d8e732e8e2bd7f5743bae17e369 (cherry picked from commit b1e82163e259305ed989b005a736f941bf183f5a) ref #18012 --- base/env.jl | 2 +- test/env.jl | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/base/env.jl b/base/env.jl index fc3fe87e78c70..3c77619400b82 100644 --- a/base/env.jl +++ b/base/env.jl @@ -94,7 +94,7 @@ end function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) pos = block[1] blk = block[2] - len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + 1 buf = Array{UInt16}(len) unsafe_copy!(pointer(buf), pos, len) env = transcode(String, buf) diff --git a/test/env.jl b/test/env.jl index 592c378cf65ee..23e8710398cab 100644 --- a/test/env.jl +++ b/test/env.jl @@ -30,3 +30,36 @@ key = randstring(25) @test !haskey(ENV,key) @test_throws KeyError ENV[key] @test get(ENV,key,"default") == "default" + +# Test for #17956 +@test length(ENV) > 1 +k1, k2 = "__test__", "__test1__" +withenv(k1=>k1, k2=>k2) do + b_k1, b_k2 = false, false + for (k, v) in ENV + if k==k1 + b_k1=true + elseif k==k2 + b_k2=true + end + end + @test b_k1 && b_k2 + io = IOBuffer() + show(io, ENV) + s = takebuf_string(io) + @test contains(s, "$k1=$k1") + @test contains(s, "$k2=$k2") + + @test pop!(ENV, k1) == k1 + @test !haskey(ENV, k1) + ENV[k1] = k1 + @test pop!(ENV, k1) == k1 + @test pop!(ENV, k1, "not_there") == "not_there" + + ENV[k1] = k1 + @test delete!(ENV, k1) == ENV + @test !haskey(ENV, k1) +end + +# Test for #10853 +@test withenv(Dict{Any,Any}()...) do; true; end From 39c7c7faffa86318d93f74afa14abe5fdee28071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Sat, 13 Aug 2016 13:50:12 +0200 Subject: [PATCH 024/114] move doc for `withenv` (cherry picked from commit bb7272ed6556ec19ffc606eb195b0bcf65bfa503) ref #18012 --- base/docs/helpdb/Base.jl | 11 ----------- base/env.jl | 10 +++++++++- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/base/docs/helpdb/Base.jl b/base/docs/helpdb/Base.jl index d510c91a7b7b2..f02519fb3f317 100644 --- a/base/docs/helpdb/Base.jl +++ b/base/docs/helpdb/Base.jl @@ -2127,17 +2127,6 @@ Largest integer less than or equal to `x/y`. """ fld -""" - withenv(f::Function, kv::Pair...) - -Execute `f()` in an environment that is temporarily modified (not replaced as in `setenv`) -by zero or more `"var"=>val` arguments `kv`. `withenv` is generally used via the -`withenv(kv...) do ... end` syntax. A value of `nothing` can be used to temporarily unset an -environment variable (if it is set). When `withenv` returns, the original environment has -been restored. -""" -withenv - """ setdiff!(s, iterable) diff --git a/base/env.jl b/base/env.jl index 3c77619400b82..b0815521531f7 100644 --- a/base/env.jl +++ b/base/env.jl @@ -139,7 +139,15 @@ function show(io::IO, ::EnvHash) end end -# temporarily set and then restore an environment value +""" + withenv(f::Function, kv::Pair...) + +Execute `f()` in an environment that is temporarily modified (not replaced as in `setenv`) +by zero or more `"var"=>val` arguments `kv`. `withenv` is generally used via the +`withenv(kv...) do ... end` syntax. A value of `nothing` can be used to temporarily unset an +environment variable (if it is set). When `withenv` returns, the original environment has +been restored. +""" function withenv{T<:AbstractString}(f::Function, keyvals::Pair{T}...) old = Dict{T,Any}() for (key,val) in keyvals From 9d56d578c61a7f58e9379e43179aa80d56f92a82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20H=C3=B8egh?= Date: Sun, 14 Aug 2016 12:51:18 +0200 Subject: [PATCH 025/114] Formatting: indent the OS specific blocks. (cherry picked from commit 319cae5641b4dec8af9697d568313577caf95fbf) ref #18012 --- base/env.jl | 152 +++++++++++++++++++++++++--------------------------- 1 file changed, 74 insertions(+), 78 deletions(-) diff --git a/base/env.jl b/base/env.jl index b0815521531f7..ee6ca3b1839fb 100644 --- a/base/env.jl +++ b/base/env.jl @@ -1,61 +1,59 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license if is_windows() -const ERROR_ENVVAR_NOT_FOUND = UInt32(203) + const ERROR_ENVVAR_NOT_FOUND = UInt32(203) -_getenvlen(var::Vector{UInt16}) = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,C_NULL,0) -_hasenv(s::Vector{UInt16}) = _getenvlen(s) != 0 || Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND -_hasenv(s::AbstractString) = _hasenv(cwstring(s)) + _getenvlen(var::Vector{UInt16}) = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,C_NULL,0) + _hasenv(s::Vector{UInt16}) = _getenvlen(s) != 0 || Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND + _hasenv(s::AbstractString) = _hasenv(cwstring(s)) -function access_env(onError::Function, str::AbstractString) - var = cwstring(str) - len = _getenvlen(var) - if len == 0 - return Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND ? "" : onError(str) + function access_env(onError::Function, str::AbstractString) + var = cwstring(str) + len = _getenvlen(var) + if len == 0 + return Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND ? "" : onError(str) + end + val = zeros(UInt16,len) + ret = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,val,len) + if (ret == 0 && len != 1) || ret != len-1 || val[end] != 0 + error(string("getenv: ", str, ' ', len, "-1 != ", ret, ": ", Libc.FormatMessage())) + end + pop!(val) # NUL + return transcode(String, val) end - val = zeros(UInt16,len) - ret = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,val,len) - if (ret == 0 && len != 1) || ret != len-1 || val[end] != 0 - error(string("getenv: ", str, ' ', len, "-1 != ", ret, ": ", Libc.FormatMessage())) + + function _setenv(svar::AbstractString, sval::AbstractString, overwrite::Bool=true) + var = cwstring(svar) + val = cwstring(sval) + if overwrite || !_hasenv(var) + ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,val) + systemerror(:setenv, ret == 0) + end end - pop!(val) # NUL - return transcode(String, val) -end -function _setenv(svar::AbstractString, sval::AbstractString, overwrite::Bool=true) - var = cwstring(svar) - val = cwstring(sval) - if overwrite || !_hasenv(var) - ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,val) + function _unsetenv(svar::AbstractString) + var = cwstring(svar) + ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,C_NULL) systemerror(:setenv, ret == 0) end -end - -function _unsetenv(svar::AbstractString) - var = cwstring(svar) - ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,C_NULL) - systemerror(:setenv, ret == 0) -end - else # !windows -_getenv(var::AbstractString) = ccall(:getenv, Cstring, (Cstring,), var) -_hasenv(s::AbstractString) = _getenv(s) != C_NULL - -function access_env(onError::Function, var::AbstractString) - val = _getenv(var) - val == C_NULL ? onError(var) : unsafe_string(val) -end + _getenv(var::AbstractString) = ccall(:getenv, Cstring, (Cstring,), var) + _hasenv(s::AbstractString) = _getenv(s) != C_NULL -function _setenv(var::AbstractString, val::AbstractString, overwrite::Bool=true) - ret = ccall(:setenv, Int32, (Cstring,Cstring,Int32), var, val, overwrite) - systemerror(:setenv, ret != 0) -end + function access_env(onError::Function, var::AbstractString) + val = _getenv(var) + val == C_NULL ? onError(var) : unsafe_string(val) + end -function _unsetenv(var::AbstractString) - ret = ccall(:unsetenv, Int32, (Cstring,), var) - systemerror(:unsetenv, ret != 0) -end + function _setenv(var::AbstractString, val::AbstractString, overwrite::Bool=true) + ret = ccall(:setenv, Int32, (Cstring,Cstring,Int32), var, val, overwrite) + systemerror(:setenv, ret != 0) + end + function _unsetenv(var::AbstractString) + ret = ccall(:unsetenv, Int32, (Cstring,), var) + systemerror(:unsetenv, ret != 0) + end end # os test ## ENV: hash interface ## @@ -83,45 +81,43 @@ setindex!(::EnvHash, v, k::AbstractString) = _setenv(k,string(v)) push!(::EnvHash, k::AbstractString, v) = setindex!(ENV, v, k) if is_windows() -start(hash::EnvHash) = (pos = ccall(:GetEnvironmentStringsW,stdcall,Ptr{UInt16},()); (pos,pos)) -function done(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) - if unsafe_load(block[1]) == 0 - ccall(:FreeEnvironmentStringsW, stdcall, Int32, (Ptr{UInt16},), block[2]) - return true + start(hash::EnvHash) = (pos = ccall(:GetEnvironmentStringsW,stdcall,Ptr{UInt16},()); (pos,pos)) + function done(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) + if unsafe_load(block[1]) == 0 + ccall(:FreeEnvironmentStringsW, stdcall, Int32, (Ptr{UInt16},), block[2]) + return true + end + return false end - return false -end -function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) - pos = block[1] - blk = block[2] - len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + 1 - buf = Array{UInt16}(len) - unsafe_copy!(pointer(buf), pos, len) - env = transcode(String, buf) - m = match(r"^(=?[^=]+)=(.*)$"s, env) - if m === nothing - error("malformed environment entry: $env") + function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) + pos = block[1] + blk = block[2] + len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + 1 + buf = Array{UInt16}(len) + unsafe_copy!(pointer(buf), pos, len) + env = transcode(String, buf) + m = match(r"^(=?[^=]+)=(.*)$"s, env) + if m === nothing + error("malformed environment entry: $env") + end + return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+len*2, blk)) end - return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+len*2, blk)) -end - else # !windows -start(::EnvHash) = 0 -done(::EnvHash, i) = (ccall(:jl_environ, Any, (Int32,), i) === nothing) + start(::EnvHash) = 0 + done(::EnvHash, i) = (ccall(:jl_environ, Any, (Int32,), i) === nothing) -function next(::EnvHash, i) - env = ccall(:jl_environ, Any, (Int32,), i) - if env === nothing - throw(BoundsError()) - end - env = env::String - m = match(r"^(.*?)=(.*)$"s, env) - if m === nothing - error("malformed environment entry: $env") + function next(::EnvHash, i) + env = ccall(:jl_environ, Any, (Int32,), i) + if env === nothing + throw(BoundsError()) + end + env = env::String + m = match(r"^(.*?)=(.*)$"s, env) + if m === nothing + error("malformed environment entry: $env") + end + return (Pair{String,String}(m.captures[1], m.captures[2]), i+1) end - return (Pair{String,String}(m.captures[1], m.captures[2]), i+1) -end - end # os-test #TODO: Make these more efficent From ff6d640bd11ca7997e45d9f9db29e5517b06f9e1 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 14 Aug 2016 11:54:11 +0200 Subject: [PATCH 026/114] specialize on type in getindex for types (cherry picked from commit c13fed6ad04f0ee402373f38093954abe946fd7f) ref #18025 --- base/array.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base/array.jl b/base/array.jl index 4eba2f531ad78..13df5ab003444 100644 --- a/base/array.jl +++ b/base/array.jl @@ -126,7 +126,7 @@ similar{N}(a::Array, T::Type, dims::Dims{N}) = Array{T,N}(dims) similar{T,N}(a::Array{T}, dims::Dims{N}) = Array{T,N}(dims) # T[x...] constructs Array{T,1} -function getindex(T::Type, vals...) +function getindex{T}(::Type{T}, vals...) a = Array{T,1}(length(vals)) @inbounds for i = 1:length(vals) a[i] = vals[i] @@ -134,10 +134,10 @@ function getindex(T::Type, vals...) return a end -getindex(T::Type) = Array{T,1}(0) -getindex(T::Type, x) = (a = Array{T,1}(1); @inbounds a[1] = x; a) -getindex(T::Type, x, y) = (a = Array{T,1}(2); @inbounds (a[1] = x; a[2] = y); a) -getindex(T::Type, x, y, z) = (a = Array{T,1}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a) +getindex{T}(::Type{T}) = (@_inline_meta; Array{T,1}(0)) +getindex{T}(::Type{T}, x) = (@_inline_meta; a = Array{T,1}(1); @inbounds a[1] = x; a) +getindex{T}(::Type{T}, x, y) = (@_inline_meta; a = Array{T,1}(2); @inbounds (a[1] = x; a[2] = y); a) +getindex{T}(::Type{T}, x, y, z) = (@_inline_meta; a = Array{T,1}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a) function getindex(::Type{Any}, vals::ANY...) a = Array{Any,1}(length(vals)) From 1d3d115a79b9002e72164b576a7bb697ce10666b Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 4 Aug 2016 23:35:28 +0200 Subject: [PATCH 027/114] Fix `@threadcall` argument passing. Fixes #17819. Also document not being allowed to call back into Julia. (cherry picked from commit aea8180b086c819d216ece16a03913d6ff8b291f) ref #18016 --- base/threadcall.jl | 1 + src/ccalltest.c | 10 ++++++++++ test/ccall.jl | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/base/threadcall.jl b/base/threadcall.jl index be3e99f324cb8..da877d9707109 100644 --- a/base/threadcall.jl +++ b/base/threadcall.jl @@ -75,6 +75,7 @@ function do_threadcall(wrapper::Function, rettype::Type, argtypes::Vector, argva y = cconvert(T, x) push!(roots, y) unsafe_store!(convert(Ptr{T}, ptr), unsafe_convert(T, y)) + ptr += sizeof(T) end # create return buffer diff --git a/src/ccalltest.c b/src/ccalltest.c index e01aafa300fb2..ccd7e1cd57fde 100644 --- a/src/ccalltest.c +++ b/src/ccalltest.c @@ -21,6 +21,7 @@ int verbose = 1; int c_int = 0; + ////////////////////////////////// // Test for proper argument register truncation @@ -520,6 +521,7 @@ JL_DLLEXPORT void finalizer_cptr(void* v) set_c_int(-1); } + ////////////////////////////////// // Turn off verbose for automated tests, leave on for debugging @@ -527,6 +529,10 @@ JL_DLLEXPORT void set_verbose(int level) { verbose = level; } + +////////////////////////////////// +// Other tests + JL_DLLEXPORT void *test_echo_p(void *p) { return p; } @@ -723,3 +729,7 @@ JL_DLLEXPORT float32x4_t test_ppc64_vec2(int64_t d1, float32x4_t a, float32x4_t } #endif + +JL_DLLEXPORT int threadcall_args(int a, int b) { + return a + b; +} diff --git a/test/ccall.jl b/test/ccall.jl index 7f0792a87fdd5..56a6b1bf3f5be 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -589,6 +589,10 @@ threadcall_test_func(x) = @test threadcall_test_func(3) == 1 @test threadcall_test_func(259) == 1 +# issue 17819 +# NOTE: can't use cfunction or reuse ccalltest Struct methods, as those call into the runtime +@test @threadcall((:threadcall_args, libccalltest), Cint, (Cint, Cint), 1, 2) == 3 + let n=3 tids = Culong[] @sync for i in 1:10^n From aeb04aeb1b9214529247fb417e0e2a44c7007c01 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 13 Aug 2016 04:32:31 -0500 Subject: [PATCH 028/114] Fix missing type parameter in _array_for This fixes a performance regression in the lucompletepiv benchmark triggered by generalizing _array_for to work with indices. This call, first introduced in b363cc70ee6f09bb4472944f8690283793c6052a, likely always triggered dynamic method dispatch. The generalization to indices seemed to make that worse. (cherry picked from commit cb623f8499b45dde6a309831a094703d0abb61d0) ref #18009 --- base/array.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index 13df5ab003444..da83d1faa0b80 100644 --- a/base/array.jl +++ b/base/array.jl @@ -288,8 +288,8 @@ else _default_eltype(itr::ANY) = Any end -_array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) -_array_for(T, itr, ::HasShape) = similar(Array{T}, indices(itr)) +_array_for{T}(::Type{T}, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer)) +_array_for{T}(::Type{T}, itr, ::HasShape) = similar(Array{T}, indices(itr)) function collect(itr::Generator) isz = iteratorsize(itr.iter) From 5381a438997f7950c929e4e0b2af4f0b5b3d1e28 Mon Sep 17 00:00:00 2001 From: WooKyoung Noh Date: Mon, 15 Aug 2016 21:08:36 +0900 Subject: [PATCH 029/114] Remove RNG from exports (#18023) (cherry picked from commit 1d49ec0c575f62b0c97932414191f3e101d0232b) --- base/random.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/random.jl b/base/random.jl index c6cf2bfdeb2c0..03e9b94c5d4f3 100644 --- a/base/random.jl +++ b/base/random.jl @@ -15,7 +15,7 @@ export srand, randsubseq,randsubseq!, shuffle,shuffle!, randperm, randcycle, - AbstractRNG, RNG, MersenneTwister, RandomDevice, + AbstractRNG, MersenneTwister, RandomDevice, GLOBAL_RNG, randjump From a0ead964518b19a6f894b3ff26dabb6018fcf9f1 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 13 Aug 2016 18:46:35 +0800 Subject: [PATCH 030/114] A trick to fix continue/break/return handling in for loop testset. Fix #17908 (cherry picked from commit f6c968dbcfb149bf393776847609bd272f11a599) ref #18011 --- base/test.jl | 22 ++++++++++++++--- test/test.jl | 69 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/base/test.jl b/base/test.jl index 7d23115354bbd..0b2009ba72465 100644 --- a/base/test.jl +++ b/base/test.jl @@ -720,6 +720,13 @@ function testset_forloop(args, testloop) # wrapped in the outer loop provided by the user tests = testloop.args[2] blk = quote + # Trick to handle `break` and `continue` in the test code before + # they can be handled properly by `finally` lowering. + if !first_iteration + pop_testset() + push!(arr, finish(ts)) + end + first_iteration = false ts = $(testsettype)($desc; $options...) push_testset(ts) try @@ -729,13 +736,20 @@ function testset_forloop(args, testloop) # error in this test set record(ts, Error(:nontest_error, :(), err, catch_backtrace())) end - pop_testset() - finish(ts) end quote arr = Array{Any,1}(0) - $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), - :(push!(arr, $blk)))) + first_iteration = true + local ts + try + $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), blk)) + finally + # Handle `return` in test body + if !first_iteration + pop_testset() + push!(arr, finish(ts)) + end + end arr end end diff --git a/test/test.jl b/test/test.jl index d168a38eeb859..7e2e8b70823af 100644 --- a/test/test.jl +++ b/test/test.jl @@ -183,6 +183,50 @@ end @test typeof(tss[1]) == Base.Test.DefaultTestSet @test typeof(tss[1].results[1]) == Base.Test.Pass +# Issue #17908 (return) +testset_depth17908 = Test.get_testset_depth() +@testset for i in 1:3 + i > 1 && return + @test i == 1 +end +# The return aborts the control flow so the expression above doesn't return a +# value. The only thing we can test is whether the testset is properly popped. +# Do not use `@test` since the issue this is testing will swallow the error. +@assert testset_depth17908 == Test.get_testset_depth() + +# Issue #17462 and Issue #17908 (break, continue) +testset_depth17462 = Test.get_testset_depth() +counter_17462_pre = 0 +counter_17462_post = 0 +tss17462 = @testset for x in [1,2,3,4] + counter_17462_pre += 1 + if x == 1 + @test counter_17462_pre == x + continue + @test false + elseif x == 3 + @test counter_17462_pre == x + break + @test false + elseif x == 4 + @test false + else + @test counter_17462_pre == x + @test x == 2 + @test counter_17462_post == 0 + end + counter_17462_post += 1 +end +# Do not use `@test` since the issue this is testing will swallow the error. +# Same for the `@assert` in the for loop below +@assert testset_depth17462 == Test.get_testset_depth() +@assert length(tss17462) == 3 +for ts17462 in tss17462 + @assert isa(ts17462, Base.Test.DefaultTestSet) +end +@test counter_17462_pre == 3 +@test counter_17462_post == 1 + # now we're done running tests with DefaultTestSet so we can go back to STDOUT redirect_stdout(OLD_STDOUT) @@ -320,28 +364,3 @@ end @test @inferred(inferrable_kwtest(1; y=1)) == 2 @test @inferred(uninferrable_kwtest(1)) == 3 @test_throws ErrorException @inferred(uninferrable_kwtest(1; y=2)) == 2 - -# Issue #17462 -counter_17462_pre = 0 -counter_17462_post = 0 -@testset for x in [1,2,3,4] - counter_17462_pre += 1 - if x == 1 - @test counter_17462_pre == x - continue - @test false - elseif x == 3 - @test counter_17462_pre == x - break - @test false - elseif x == 4 - @test false - else - @test counter_17462_pre == x - @test x == 2 - @test counter_17462_post == 0 - end - counter_17462_post += 1 -end -@test counter_17462_pre == 3 -@test counter_17462_post == 1 From 14854d7e2f02bc13526a39300cd5aa4c5eb3fbae Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 10 Aug 2016 15:33:22 -0400 Subject: [PATCH 031/114] preserve functionality of special ^ inliner in linear IR (cherry picked from commit c6c46b60658065d20f4e083a9d25214d381d9286) ref #17949 --- base/inference.jl | 53 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 92874f6f7b813..8ef4403ef703a 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2812,8 +2812,6 @@ function mk_tuplecall(args, sv::InferenceState) e end -const corenumtype = Union{Int32,Int64,Float32,Float64} - function inlining_pass!(linfo::LambdaInfo, sv::InferenceState) eargs = linfo.code i = 1 @@ -2834,6 +2832,8 @@ function inlining_pass!(linfo::LambdaInfo, sv::InferenceState) end end +const corenumtype = Union{Int32, Int64, Float32, Float64} + function inlining_pass(e::Expr, sv, linfo) if e.head === :method # avoid running the inlining pass on function definitions @@ -2923,20 +2923,41 @@ function inlining_pass(e::Expr, sv, linfo) end end - if sv.inlining && isdefined(Main, :Base) && - ((isdefined(Main.Base, :^) && is(f, Main.Base.:^)) || - (isdefined(Main.Base, :.^) && is(f, Main.Base.:.^))) - if length(e.args) == 3 && isa(e.args[3],Union{Int32,Int64}) - a1 = e.args[2] - basenumtype = Union{corenumtype, Main.Base.Complex64, Main.Base.Complex128, Main.Base.Rational} - if isa(a1,basenumtype) || ((isa(a1,Symbol) || isa(a1,Slot) || isa(a1,SSAValue)) && - exprtype(a1,sv) ⊑ basenumtype) - if e.args[3]==2 - e.args = Any[GlobalRef(Main.Base,:*), a1, a1] - f = Main.Base.:*; ft = abstract_eval_constant(f) - elseif e.args[3]==3 - e.args = Any[GlobalRef(Main.Base,:*), a1, a1, a1] - f = Main.Base.:*; ft = abstract_eval_constant(f) + if sv.inlining + if isdefined(Main, :Base) && + ((isdefined(Main.Base, :^) && is(f, Main.Base.:^)) || + (isdefined(Main.Base, :.^) && is(f, Main.Base.:.^))) && + length(e.args) == 3 + + a2 = e.args[3] + if isa(a2, Symbol) || isa(a2, Slot) || isa(a2, SSAValue) + ta2 = exprtype(a2, sv) + if isa(ta2, Const) + a2 = ta2.val + end + end + + square = (a2 === Int32(2) || a2 === Int64(2)) + triple = (a2 === Int32(3) || a2 === Int64(3)) + if square || triple + a1 = e.args[2] + basenumtype = Union{corenumtype, Main.Base.Complex64, Main.Base.Complex128, Main.Base.Rational} + if isa(a1, basenumtype) || ((isa(a1, Symbol) || isa(a1, Slot) || isa(a1, SSAValue)) && + exprtype(a1, sv) ⊑ basenumtype) + if square + e.args = Any[GlobalRef(Main.Base,:*), a1, a1] + res = inlining_pass(e, sv, linfo) + else + e.args = Any[GlobalRef(Main.Base,:*), Expr(:call, GlobalRef(Main.Base,:*), a1, a1), a1] + res = inlining_pass(e, sv, linfo) + end + if isa(res, Tuple) + if isa(res[2], Array) && !isempty(res[2]) + append!(stmts, res[2]) + end + res = res[1] + end + return (res, stmts) end end end From 9a6e884f9f1bcee065fdb2705a687d521f2e4aa6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 10 Aug 2016 15:56:19 -0400 Subject: [PATCH 032/114] add model of special case ^ inliner to inference edges required for the inliner to work, since all of the logic for computing edges is supposed to be handle by the inference step fix #17759 (cherry picked from commit 19b1276eec40900ccf27ae2976fe9a6821b0668c) ref #17949 --- base/inference.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/base/inference.jl b/base/inference.jl index 8ef4403ef703a..779e99d16efe4 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1052,6 +1052,23 @@ function abstract_call(f::ANY, fargs, argtypes::Vector{Any}, vtypes::VarTable, s return Type end + if sv.inlining + # need to model the special inliner for ^ + # to ensure we have added the same edge + if isdefined(Main, :Base) && + ((isdefined(Main.Base, :^) && is(f, Main.Base.:^)) || + (isdefined(Main.Base, :.^) && is(f, Main.Base.:.^))) && + length(argtypes) == 3 && (argtypes[3] ⊑ Int32 || argtypes[3] ⊑ Int64) + + a1 = argtypes[2] + basenumtype = Union{corenumtype, Main.Base.Complex64, Main.Base.Complex128, Main.Base.Rational} + if a1 ⊑ basenumtype + ftimes = Main.Base.:* + ta1 = widenconst(a1) + abstract_call_gf_by_type(ftimes, Tuple{typeof(ftimes), ta1, ta1}, sv) + end + end + end return abstract_call_gf_by_type(f, atype, sv) end From cd44917d5777400093141e70e9e320bbb574d622 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 13 Aug 2016 13:09:41 -0400 Subject: [PATCH 033/114] fix effect_free computation over getfield fix #18015 (cherry picked from commit 72a13edf9bdc96e1c6cfc914fc798d8d255e42ca) ref #18017 --- base/inference.jl | 183 +++++++++++++++++++++------------------------- test/inference.jl | 18 ++++- 2 files changed, 102 insertions(+), 99 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 779e99d16efe4..869b30aa0f3e2 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1106,7 +1106,7 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState) if isa(e,QuoteNode) return abstract_eval_constant((e::QuoteNode).value) elseif isa(e,SSAValue) - return abstract_eval_ssavalue(e::SSAValue, sv) + return abstract_eval_ssavalue(e::SSAValue, sv.linfo) elseif isa(e,Slot) return vtypes[e.id].typ elseif isa(e,Symbol) @@ -1190,8 +1190,8 @@ function abstract_eval_global(M::Module, s::Symbol) return Any end -function abstract_eval_ssavalue(s::SSAValue, sv::InferenceState) - typ = sv.linfo.ssavaluetypes[s.id+1] +function abstract_eval_ssavalue(s::SSAValue, linfo::LambdaInfo) + typ = linfo.ssavaluetypes[s.id + 1] if typ === NF return Bottom end @@ -1921,7 +1921,7 @@ function finish(me::InferenceState) if !ispure && length(me.linfo.code) < 10 ispure = true for stmt in me.linfo.code - if !statement_effect_free(stmt, me) + if !statement_effect_free(stmt, me.linfo) ispure = false; break end end @@ -2177,20 +2177,21 @@ function occurs_more(e::ANY, pred, n) return 0 end -function exprtype(x::ANY, sv::InferenceState) - if isa(x,Expr) +function exprtype(x::ANY, linfo::LambdaInfo) + if isa(x, Expr) return (x::Expr).typ - elseif isa(x,SlotNumber) - return sv.linfo.slottypes[x.id] - elseif isa(x,TypedSlot) + elseif isa(x, SlotNumber) + return linfo.slottypes[x.id] + elseif isa(x, TypedSlot) return (x::Slot).typ - elseif isa(x,SSAValue) - return abstract_eval_ssavalue(x::SSAValue, sv) - elseif isa(x,Symbol) - return abstract_eval_global(sv.mod, x::Symbol) - elseif isa(x,QuoteNode) + elseif isa(x, SSAValue) + return abstract_eval_ssavalue(x::SSAValue, linfo) + elseif isa(x, Symbol) + mod = isdefined(linfo, :def) ? linfo.def.module : current_module() + return abstract_eval_global(mod, x::Symbol) + elseif isa(x, QuoteNode) return abstract_eval_constant((x::QuoteNode).value) - elseif isa(x,GlobalRef) + elseif isa(x, GlobalRef) return abstract_eval_global(x.mod, (x::GlobalRef).name) else return abstract_eval_constant(x) @@ -2221,28 +2222,28 @@ function is_pure_builtin(f::ANY) return false end -function statement_effect_free(e::ANY, sv) - if isa(e,Expr) +function statement_effect_free(e::ANY, linfo::LambdaInfo) + if isa(e, Expr) if e.head === :(=) - return !isa(e.args[1],GlobalRef) && effect_free(e.args[2], sv, false) + return !isa(e.args[1], GlobalRef) && effect_free(e.args[2], linfo, false) elseif e.head === :gotoifnot - return effect_free(e.args[1], sv, false) + return effect_free(e.args[1], linfo, false) end - elseif isa(e,LabelNode) || isa(e,GotoNode) + elseif isa(e, LabelNode) || isa(e, GotoNode) return true end - return effect_free(e, sv, false) + return effect_free(e, linfo, false) end # detect some important side-effect-free calls (allow_volatile=true) # and some affect-free calls (allow_volatile=false) -- affect_free means the call # cannot be affected by previous calls, except assignment nodes -function effect_free(e::ANY, sv, allow_volatile::Bool) - if isa(e,GlobalRef) +function effect_free(e::ANY, linfo::LambdaInfo, allow_volatile::Bool) + if isa(e, GlobalRef) return (isdefined(e.mod, e.name) && (allow_volatile || isconst(e.mod, e.name))) - elseif isa(e,Symbol) + elseif isa(e, Symbol) return allow_volatile - elseif isa(e,Expr) + elseif isa(e, Expr) e = e::Expr head = e.head if head === :static_parameter || head === :meta || head === :line || @@ -2251,32 +2252,17 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) end ea = e.args if head === :call && !isa(e.args[1], SSAValue) && !isa(e.args[1], Slot) - if is_known_call_p(e, is_pure_builtin, sv) + if is_known_call_p(e, is_pure_builtin, linfo) if !allow_volatile - if is_known_call(e, arrayref, sv) || is_known_call(e, arraylen, sv) + if is_known_call(e, arrayref, linfo) || is_known_call(e, arraylen, linfo) return false - elseif is_known_call(e, getfield, sv) - # arguments must be immutable to ensure e is affect_free - first = true - for a in ea - if first # first "arg" is the function name - first = false - continue - end - if isa(a,Symbol) - return false - end - if isa(a,SSAValue) - typ = widenconst(exprtype(a,sv)) - if !isa(typ,DataType) || typ.mutable - return false - end - end - if !effect_free(a,sv,allow_volatile) - return false - end + elseif is_known_call(e, getfield, linfo) + # first argument must be immutable to ensure e is affect_free + a = ea[2] + typ = widenconst(exprtype(a, linfo)) + if !isa(typ, DataType) || typ.mutable || typ.abstract + return false end - return true end end # fall-through @@ -2286,7 +2272,7 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) elseif head === :new if !allow_volatile a = ea[1] - typ = widenconst(exprtype(a,sv)) + typ = widenconst(exprtype(a, linfo)) if !isType(typ) || !isa((typ::Type).parameters[1],DataType) || ((typ::Type).parameters[1]::DataType).mutable return false end @@ -2300,11 +2286,11 @@ function effect_free(e::ANY, sv, allow_volatile::Bool) return false end for a in ea - if !effect_free(a,sv,allow_volatile) + if !effect_free(a, linfo, allow_volatile) return false end end - elseif isa(e,LabelNode) || isa(e,GotoNode) + elseif isa(e, LabelNode) || isa(e, GotoNode) return false end return true @@ -2313,12 +2299,12 @@ end #### post-inference optimizations #### -function inline_as_constant(val::ANY, argexprs, sv) +function inline_as_constant(val::ANY, argexprs, linfo::LambdaInfo) # check if any arguments aren't effect_free and need to be kept around stmts = Any[] for i = 1:length(argexprs) arg = argexprs[i] - if !effect_free(arg, sv, false) + if !effect_free(arg, linfo, false) push!(stmts, arg) end end @@ -2365,7 +2351,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference istopfunction(topmod, f, :typejoin) || istopfunction(topmod, f, :promote_type)) # XXX: compute effect_free for the actual arguments - if length(argexprs) < 2 || effect_free(argexprs[2], sv, true) + if length(argexprs) < 2 || effect_free(argexprs[2], enclosing, true) return (e.typ.parameters[1],()) else return (e.typ.parameters[1], Any[argexprs[2]]) @@ -2373,11 +2359,11 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end end if istopfunction(topmod, f, :isbits) && length(atypes)==2 && isType(atypes[2]) && - effect_free(argexprs[2],sv,true) && isleaftype(atypes[2].parameters[1]) + effect_free(argexprs[2], enclosing, true) && isleaftype(atypes[2].parameters[1]) return (isbits(atypes[2].parameters[1]),()) end if is(f, Core.kwfunc) && length(argexprs) == 2 && isa(e.typ, Const) - if effect_free(argexprs[2], sv, true) + if effect_free(argexprs[2], enclosing, true) return (e.typ.val, ()) else return (e.typ.val, Any[argexprs[2]]) @@ -2407,7 +2393,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference local ti = atypes[i] if arg_hoisted || isa(ti, Union) aei = ex.args[i] - if !effect_free(aei, sv, false) + if !effect_free(aei, enclosing, false) arg_hoisted = true newvar = newvar!(sv, ti) insert!(stmts, 1, Expr(:(=), newvar, aei)) @@ -2516,11 +2502,11 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference (isType(e.typ) || isa(e.typ,Const)) if isType(e.typ) if !has_typevars(e.typ.parameters[1]) - return inline_as_constant(e.typ.parameters[1], argexprs, sv) + return inline_as_constant(e.typ.parameters[1], argexprs, enclosing) end else assert(isa(e.typ,Const)) - return inline_as_constant(e.typ.val, argexprs, sv) + return inline_as_constant(e.typ.val, argexprs, enclosing) end end @@ -2535,7 +2521,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference end if linfo !== nothing && linfo.jlcall_api == 2 # in this case function can be inlined to a constant - return inline_as_constant(linfo.constval, argexprs, sv) + return inline_as_constant(linfo.constval, argexprs, enclosing) elseif linfo !== nothing && !linfo.inlineable return invoke_NF() elseif linfo === nothing || linfo.code === nothing @@ -2601,7 +2587,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference for i=na:-1:1 # stmts_free needs to be calculated in reverse-argument order #args_i = args[i] aei = argexprs[i] - aeitype = argtype = widenconst(exprtype(aei,sv)) + aeitype = argtype = widenconst(exprtype(aei, enclosing)) # ok for argument to occur more than once if the actual argument # is a symbol or constant, or is not affected by previous statements @@ -2613,17 +2599,17 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference if occ < 6 occ += occurs_more(b, x->(isa(x,Slot)&&x.id==i), 6) end - # TODO: passing `sv` here is wrong since it refers to the enclosing function - if occ > 0 && affect_free && !effect_free(b, sv, true) #TODO: we could short-circuit this test better by memoizing effect_free(b) in the for loop over i + if occ > 0 && affect_free && !effect_free(b, linfo, true) + #TODO: we might be able to short-circuit this test better by memoizing effect_free(b) in the for loop over i affect_free = false end if occ > 5 && !affect_free break end end - free = effect_free(aei,sv,true) + free = effect_free(aei, enclosing, true) if ((occ==0 && is(aeitype,Bottom)) || (occ > 1 && !inline_worthy(aei, occ*2000)) || - (affect_free && !free) || (!affect_free && !effect_free(aei,sv,false))) + (affect_free && !free) || (!affect_free && !effect_free(aei, enclosing, false))) if occ != 0 vnew = newvar!(sv, aeitype) argexprs[i] = vnew @@ -2825,8 +2811,8 @@ end function mk_tuplecall(args, sv::InferenceState) e = Expr(:call, top_tuple, args...) - e.typ = tuple_tfunc(Tuple{Any[widenconst(exprtype(x,sv)) for x in args]...}) - e + e.typ = tuple_tfunc(Tuple{Any[widenconst(exprtype(x, sv.linfo)) for x in args]...}) + return e end function inlining_pass!(linfo::LambdaInfo, sv::InferenceState) @@ -2865,11 +2851,11 @@ function inlining_pass(e::Expr, sv, linfo) # don't inline first (global) arguments of ccall, as this needs to be evaluated # by the interpreter and inlining might put in something it can't handle, # like another ccall (or try to move the variables out into the function) - if is_known_call(e, Core.Intrinsics.ccall, sv) + if is_known_call(e, Core.Intrinsics.ccall, linfo) # 4 is rewritten to 2 below to handle the callee. i0 = 4 isccall = true - elseif is_known_call(e, Core.Intrinsics.llvmcall, sv) + elseif is_known_call(e, Core.Intrinsics.llvmcall, linfo) i0 = 5 isccall = false else @@ -2895,8 +2881,8 @@ function inlining_pass(e::Expr, sv, linfo) end res = inlining_pass(ei, sv, linfo) res1 = res[1] - if has_stmts && !effect_free(res1, sv, false) - restype = exprtype(res1,sv) + if has_stmts && !effect_free(res1, linfo, false) + restype = exprtype(res1, linfo) vnew = newvar!(sv, restype) argloc[i] = vnew unshift!(stmts, Expr(:(=), vnew, res1)) @@ -2909,7 +2895,7 @@ function inlining_pass(e::Expr, sv, linfo) prepend!(stmts,res2) if !has_stmts for stmt in res2 - if !effect_free(stmt, sv, true) + if !effect_free(stmt, linfo, true) has_stmts = true end end @@ -2930,7 +2916,7 @@ function inlining_pass(e::Expr, sv, linfo) end end - ft = exprtype(arg1, sv) + ft = exprtype(arg1, linfo) if isa(ft, Const) f = ft.val else @@ -2948,7 +2934,7 @@ function inlining_pass(e::Expr, sv, linfo) a2 = e.args[3] if isa(a2, Symbol) || isa(a2, Slot) || isa(a2, SSAValue) - ta2 = exprtype(a2, sv) + ta2 = exprtype(a2, linfo) if isa(ta2, Const) a2 = ta2.val end @@ -2960,7 +2946,7 @@ function inlining_pass(e::Expr, sv, linfo) a1 = e.args[2] basenumtype = Union{corenumtype, Main.Base.Complex64, Main.Base.Complex128, Main.Base.Rational} if isa(a1, basenumtype) || ((isa(a1, Symbol) || isa(a1, Slot) || isa(a1, SSAValue)) && - exprtype(a1, sv) ⊑ basenumtype) + exprtype(a1, linfo) ⊑ basenumtype) if square e.args = Any[GlobalRef(Main.Base,:*), a1, a1] res = inlining_pass(e, sv, linfo) @@ -2984,7 +2970,7 @@ function inlining_pass(e::Expr, sv, linfo) ata = Vector{Any}(length(e.args)) ata[1] = ft for i = 2:length(e.args) - a = exprtype(e.args[i], sv) + a = exprtype(e.args[i], linfo) (a === Bottom || isvarargtype(a)) && return (e, stmts) ata[i] = a end @@ -2999,7 +2985,7 @@ function inlining_pass(e::Expr, sv, linfo) if !is(res,NF) # iteratively inline apply(f, tuple(...), tuple(...), ...) in order # to simplify long vararg lists as in multi-arg + - if isa(res,Expr) && is_known_call(res, _apply, sv) + if isa(res,Expr) && is_known_call(res, _apply, linfo) e = res::Expr f = _apply; ft = abstract_eval_constant(f) else @@ -3007,18 +2993,19 @@ function inlining_pass(e::Expr, sv, linfo) end end - if is(f,_apply) + if is(f, _apply) na = length(e.args) newargs = Vector{Any}(na-2) for i = 3:na aarg = e.args[i] - t = widenconst(exprtype(aarg,sv)) - if isa(aarg,Expr) && (is_known_call(aarg, tuple, sv) || is_known_call(aarg, svec, sv)) + t = widenconst(exprtype(aarg, linfo)) + if isa(aarg,Expr) && (is_known_call(aarg, tuple, linfo) || is_known_call(aarg, svec, linfo)) # apply(f,tuple(x,y,...)) => f(x,y,...) newargs[i-2] = aarg.args[2:end] elseif isa(aarg, Tuple) newargs[i-2] = Any[ QuoteNode(x) for x in aarg ] - elseif isa(t,DataType) && t.name===Tuple.name && !isvatuple(t) && effect_free(aarg,sv,true) && length(t.parameters) <= MAX_TUPLE_SPLAT + elseif isa(t, DataType) && t.name === Tuple.name && !isvatuple(t) && + effect_free(aarg, linfo, true) && length(t.parameters) <= MAX_TUPLE_SPLAT # apply(f,t::(x,y)) => f(t[1],t[2]) tp = t.parameters newargs[i-2] = Any[ mk_getfield(aarg,j,tp[j]) for j=1:length(tp) ] @@ -3030,7 +3017,7 @@ function inlining_pass(e::Expr, sv, linfo) e.args = [Any[e.args[2]]; newargs...] # now try to inline the simplified call - ft = exprtype(e.args[1], sv) + ft = exprtype(e.args[1], linfo) if isa(ft,Const) f = ft.val else @@ -3053,23 +3040,23 @@ function add_slot!(linfo::LambdaInfo, typ, is_sa, name=compiler_temp_sym) push!(linfo.slotnames, name) push!(linfo.slottypes, typ) push!(linfo.slotflags, Slot_Assigned + is_sa * Slot_AssignedOnce) - SlotNumber(id) + return SlotNumber(id) end -function is_known_call(e::Expr, func, sv) +function is_known_call(e::Expr, func::ANY, linfo::LambdaInfo) if e.head !== :call return false end - f = exprtype(e.args[1], sv) - return isa(f,Const) && f.val === func + f = exprtype(e.args[1], linfo) + return isa(f, Const) && f.val === func end -function is_known_call_p(e::Expr, pred, sv) +function is_known_call_p(e::Expr, pred::ANY, linfo::LambdaInfo) if e.head !== :call return false end - f = exprtype(e.args[1], sv) - return isa(f,Const) && pred(f.val) + f = exprtype(e.args[1], linfo) + return isa(f, Const) && pred(f.val) end function delete_var!(linfo, id, T) @@ -3172,7 +3159,7 @@ function occurs_outside_getfield(linfo::LambdaInfo, e::ANY, sym::ANY, end if isa(e,Expr) e = e::Expr - if is_known_call(e, getfield, sv) && symequal(e.args[2],sym) + if is_known_call(e, getfield, linfo) && symequal(e.args[2],sym) idx = e.args[3] if isa(idx,QuoteNode) && (idx.value in field_names) return false @@ -3228,7 +3215,7 @@ function _getfield_elim_pass!(e::Expr, sv) for i = 1:length(e.args) e.args[i] = _getfield_elim_pass!(e.args[i], sv) end - if is_known_call(e, getfield, sv) && length(e.args)==3 && + if is_known_call(e, getfield, sv.linfo) && length(e.args)==3 && (isa(e.args[3],Int) || isa(e.args[3],QuoteNode)) e1 = e.args[2] j = e.args[3] @@ -3243,7 +3230,7 @@ function _getfield_elim_pass!(e::Expr, sv) ok = true for k = 2:length(e1.args) k == j+1 && continue - if !effect_free(e1.args[k], sv, true) + if !effect_free(e1.args[k], sv.linfo, true) ok = false; break end end @@ -3273,10 +3260,10 @@ _getfield_elim_pass!(e::ANY, sv) = e # getfield(..., 1 <= x <= n) or getfield(..., x in f) on the result function is_allocation(e :: ANY, sv::InferenceState) isa(e, Expr) || return false - if is_known_call(e, tuple, sv) + if is_known_call(e, tuple, sv.linfo) return (length(e.args)-1,()) elseif e.head === :new - typ = widenconst(exprtype(e, sv)) + typ = widenconst(exprtype(e, sv.linfo)) if isleaftype(typ) @assert(isa(typ,DataType)) nf = length(e.args)-1 @@ -3305,7 +3292,7 @@ function gotoifnot_elim_pass!(linfo::LambdaInfo, sv::InferenceState) expr = expr::Expr expr.head === :gotoifnot || continue cond = expr.args[1] - condt = exprtype(cond, sv) + condt = exprtype(cond, linfo) isa(condt, Const) || continue val = (condt::Const).val # Codegen should emit an unreachable if val is not a Bool so @@ -3382,7 +3369,7 @@ function alloc_elim_pass!(linfo::LambdaInfo, sv::InferenceState) isa(tupelt,QuoteNode) || isa(tupelt, SSAValue)) vals[j] = tupelt else - elty = exprtype(tupelt,sv) + elty = exprtype(tupelt, linfo) if is_ssa tmpv = newvar!(sv, elty) else @@ -3437,7 +3424,7 @@ end function replace_getfield!(linfo::LambdaInfo, e::Expr, tupname, vals, field_names, sv) for i = 1:length(e.args) a = e.args[i] - if isa(a,Expr) && is_known_call(a, getfield, sv) && + if isa(a,Expr) && is_known_call(a, getfield, linfo) && symequal(a.args[2],tupname) idx = if isa(a.args[3], Int) a.args[3] @@ -3460,7 +3447,7 @@ function replace_getfield!(linfo::LambdaInfo, e::Expr, tupname, vals, field_name end elseif isa(val,SSAValue) val = val::SSAValue - typ = exprtype(val, sv) + typ = exprtype(val, linfo) if a.typ ⊑ typ && !(typ ⊑ a.typ) sv.linfo.ssavaluetypes[val.id+1] = a.typ end diff --git a/test/inference.jl b/test/inference.jl index 1d99d6539bc14..f1f4fbb9b1457 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -68,7 +68,7 @@ end abstract Outer5906{T} immutable Inner5906{T} - a:: T + a:: T end immutable Empty5906{T} <: Outer5906{T} @@ -306,3 +306,19 @@ let T = Array{Tuple{Vararg{Float64,TypeVar(:dim)}},1}, @test Base.return_types(f16530b, (Symbol,)) == Any[TTlim] end @test f16530a(:d) == Vector + + +# issue #18015 +type Triple18015 + a::Int + b::Int + c::Int +end +a18015(tri) = tri.a +b18015(tri) = tri.b +c18015(tri) = tri.c +setabc18015!(tri, a, b, c) = (tri.a = a; tri.b = b; tri.c = c) +let tri = Triple18015(1, 2, 3) + setabc18015!(tri, b18015(tri), c18015(tri), a18015(tri)) + @test tri.a === 2 && tri.b === 3 && tri.c === 1 +end From 1d566a977d730a027e3a7c1a53f3eb57958c22c9 Mon Sep 17 00:00:00 2001 From: Anders Claesson Date: Mon, 15 Aug 2016 15:15:02 +0000 Subject: [PATCH 034/114] Fix a trivial typo in devdocs (cherry picked from commit 854226571b3e2bda2019f97cbff8ff575571ab90) ref #18033 --- doc/devdocs/object.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/devdocs/object.rst b/doc/devdocs/object.rst index 38c31075ddc51..99137ea5be094 100644 --- a/doc/devdocs/object.rst +++ b/doc/devdocs/object.rst @@ -43,7 +43,7 @@ the values can also be extracted directly as an array access:: As an example, a "boxed" :c:type:`uint16_t` is stored as follows:: struct { - oqaque metadata; + opaque metadata; struct { uint16_t data; -- 2 bytes } jl_value_t; From 20673d9caa4e40219d516435404dcd3f059251dd Mon Sep 17 00:00:00 2001 From: Twan Koolen Date: Mon, 15 Aug 2016 13:08:54 -0400 Subject: [PATCH 035/114] Make Colon immutable. Fixes 18034. (cherry picked from commit 06b171dafcd18bc7d5da0d7c137a09fe698db417) ref #18036 --- base/essentials.jl | 2 +- test/subarray.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/base/essentials.jl b/base/essentials.jl index fed6c191ea4de..30b8d83148f46 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -202,7 +202,7 @@ function isassigned(v::SimpleVector, i::Int) end # index colon -type Colon +immutable Colon end const (:) = Colon() diff --git a/test/subarray.jl b/test/subarray.jl index 0ca8b0b05c858..2cfc8d772955d 100644 --- a/test/subarray.jl +++ b/test/subarray.jl @@ -484,3 +484,15 @@ u = (1,2:3) let size=(x,y)-> error("should not happen") @test X[1:end,2,2] == @view X[1:end,2,2] end + +# issue #18034 +# ensure that it is possible to create an isbits, LinearFast view of an immutable Array +let + immutable ImmutableTestArray{T, N} <: Base.DenseArray{T, N} + end + Base.size(::Union{ImmutableTestArray, Type{ImmutableTestArray}}) = (0, 0) + Base.linearindexing(::Union{ImmutableTestArray, Type{ImmutableTestArray}}) = Base.LinearFast() + a = ImmutableTestArray{Float64, 2}() + @test Base.linearindexing(view(a, :, :)) == Base.LinearFast() + @test isbits(view(a, :, :)) +end From bc19ea404ecc726e32cf95d8acf105cf19fca88d Mon Sep 17 00:00:00 2001 From: ranjanan Date: Fri, 12 Aug 2016 19:23:06 +0530 Subject: [PATCH 036/114] Write up test for #15592 parse("[1] [2]") should throw a ParseError (cherry picked from commit 55e016d2f6175b4a3bad73bb9146e6ee8c36ad5f) ref #17988 --- test/parse.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/parse.jl b/test/parse.jl index f33e55273118e..3c625c192dc8d 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -662,3 +662,8 @@ end # issue #17701 @test expand(:(i==3 && i+=1)) == Expr(:error, "invalid assignment location \"==(i,3)&&i\"") + +# PR #15592 +let str = "[1] [2]" + @test_throws ParseError parse(str) +end From c308b91bbcb50f189d5d3af4768f83e91f5905d6 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 11 Aug 2016 14:16:11 -0500 Subject: [PATCH 037/114] Expand docs on enumerate's relationship with indexing (cherry picked from commit 2311507718e68f4437bd570d53a24f1e7158b12f) ref #17978 --- base/iterator.jl | 10 ++++++---- doc/stdlib/collections.rst | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/base/iterator.jl b/base/iterator.jl index 09dca34b880e4..715ff753bb0fa 100644 --- a/base/iterator.jl +++ b/base/iterator.jl @@ -21,10 +21,12 @@ end """ enumerate(iter) -An iterator that yields `(i, x)` where `i` is an index starting at 1, and -`x` is the `i`th value from the given iterator. It's useful when you need -not only the values `x` over which you are iterating, but also the index `i` -of the iterations. +An iterator that yields `(i, x)` where `i` is a counter starting at 1, +and `x` is the `i`th value from the given iterator. It's useful when +you need not only the values `x` over which you are iterating, but +also the number of iterations so far. Note that `i` may not be valid +for indexing `iter`; it's also possible that `x != iter[i]`, if `iter` +has indices that do not start at 1. ```jldoctest julia> a = ["a", "b", "c"]; diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 59abf1b78aba2..3c2eac2380a99 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -81,7 +81,7 @@ type. .. Docstring generated from Julia source - An iterator that yields ``(i, x)`` where ``i`` is an index starting at 1, and ``x`` is the ``i``\ th value from the given iterator. It's useful when you need not only the values ``x`` over which you are iterating, but also the index ``i`` of the iterations. + An iterator that yields ``(i, x)`` where ``i`` is a counter starting at 1, and ``x`` is the ``i``\ th value from the given iterator. It's useful when you need not only the values ``x`` over which you are iterating, but also the number of iterations so far. Note that ``i`` may not be valid for indexing ``iter``\ ; it's also possible that ``x != iter[i]``\ , if ``iter`` has indices that do not start at 1. .. doctest:: From 1c255479e33d9489ac76eac19c7babaf3c3d4514 Mon Sep 17 00:00:00 2001 From: Mus M Date: Tue, 16 Aug 2016 01:33:45 -0400 Subject: [PATCH 038/114] Update modules.rst adding type to show for v0.5 (#18018) Update to add type annotation otherwise we get ``` ERROR: MethodError: no method matching display(::MyType) ``` (cherry picked from commit 5076870387429448cddc8850aea442218b8d670d) --- doc/manual/modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/modules.rst b/doc/manual/modules.rst index aab14db34fb52..72241c2ea00a7 100644 --- a/doc/manual/modules.rst +++ b/doc/manual/modules.rst @@ -36,7 +36,7 @@ not meant to be run, but is shown for illustrative purposes:: bar(x) = 2x foo(a::MyType) = bar(a.x) + 1 - show(io, a::MyType) = print(io, "MyType $(a.x)") + show(io::IO, a::MyType) = print(io, "MyType $(a.x)") end Note that the style is not to indent the body of the module, since From c144c184766b682acfdc9f6adde631c8dba22f83 Mon Sep 17 00:00:00 2001 From: Michele Zaffalon Date: Tue, 16 Aug 2016 07:34:28 +0200 Subject: [PATCH 039/114] added example of array type conversion (#17663) * added example of array type conversion ... because it seems to bite less experienced users like myself. (Stolen from Terry Seaward's email, https://groups.google.com/d/msg/julia-users/wzaJw_FfNlA/qcuZ4FggEL4J) * added example of array type conversion (cherry picked from commit 62615c374d348a72f000e956cddb0a7864b2a471) --- doc/manual/conversion-and-promotion.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 61969179addc5..11651893358c3 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -90,6 +90,16 @@ action: julia> typeof(ans) Float64 + julia> a = Any[1 2 3; 4 5 6] + 2x3 Array{Any,2}: + 1 2 3 + 4 5 6 + + julia> convert(Array{Float64}, a) + 2x3 Array{Float64,2}: + 1.0 2.0 3.0 + 4.0 5.0 6.0 + Conversion isn't always possible, in which case a no method error is thrown indicating that ``convert`` doesn't know how to perform the requested conversion: From 0136e4f72a6649c64d64ecb0e9faa0741ce237c8 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 15 Aug 2016 15:37:31 -0500 Subject: [PATCH 040/114] Make `find` indices-aware (cherry picked from commit 6f1ad9c11cb8d91f1f1ac0cae8d03f532ba54b62) ref #18040 --- base/array.jl | 8 ++++++-- test/offsetarray.jl | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index da83d1faa0b80..2e0f5b4feb98b 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1077,15 +1077,18 @@ function find(testf::Function, A) # use a dynamic-length array to store the indexes, then copy to a non-padded # array for the return tmpI = Array{Int,1}(0) + inds = _index_remapper(A) for (i,a) = enumerate(A) if testf(a) - push!(tmpI, i) + push!(tmpI, inds[i]) end end I = Array{Int,1}(length(tmpI)) copy!(I, tmpI) return I end +_index_remapper(A::AbstractArray) = linearindices(A) +_index_remapper(iter) = Colon() # safe for objects that don't implement length """ find(A) @@ -1110,9 +1113,10 @@ function find(A) nnzA = countnz(A) I = Vector{Int}(nnzA) count = 1 + inds = _index_remapper(A) for (i,a) in enumerate(A) if a != 0 - I[count] = i + I[count] = inds[i] count += 1 end end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 2136e4dd580e0..d1a370a3d67f5 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -281,6 +281,11 @@ I,J,N = findnz(z) @test I == [-1] @test J == [0] @test N == [2] +h = OffsetArray([-1,1,-2,2,0], (-3,)) +@test find(h) == [-2:1;] +@test find(x->x>0, h) == [-1,1] +@test find(x->x<0, h) == [-2,0] +@test find(x->x==0, h) == [2] v = OffsetArray([1,1e100,1,-1e100], (-3,))*1000 v2 = OffsetArray([1,-1e100,1,1e100], (5,))*1000 From 18aa36ebd4d13cad347f9068ff57167a7e9c9e73 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 15 Aug 2016 15:44:51 -0500 Subject: [PATCH 041/114] Fixes and more tests for indexing with non-1 logical arrays (cherry picked from commit c49147366cfde959c36c8847f90596f071edd95d) ref #18040 --- base/multidimensional.jl | 2 +- test/offsetarray.jl | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 3773db8701964..660773dbe1790 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -318,7 +318,7 @@ function _unsafe_getindex(::LinearFast, src::AbstractArray, I::AbstractArray{Boo D = eachindex(dest) Ds = start(D) - s = 0 + s = first(linearindices(src))-1 for i in eachindex(I) s += 1 @inbounds if I[i] diff --git a/test/offsetarray.jl b/test/offsetarray.jl index d1a370a3d67f5..2bb5104a88103 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -7,6 +7,7 @@ let # Basics v0 = rand(4) v = OffsetArray(v0, (-3,)) +h = OffsetArray([-1,1,-2,2,0], (-3,)) @test indices(v) == (-2:1,) @test_throws ErrorException size(v) @test_throws ErrorException size(v, 1) @@ -48,6 +49,16 @@ S = OffsetArray(view(A0, 1:2, 1:2), (-1,2)) # LinearSlow @test eachindex(A) == 1:4 @test eachindex(S) == CartesianRange((0:1,3:4)) +# logical indexing +@test A[A .> 2] == [3,4] +@test_throws BoundsError h[trues(2)] +@test_throws BoundsError h[trues(5)] +@test h[OffsetArray(trues(5), (-3,))] == parent(h) +@test h[OffsetArray([true,false,false,true,true], (-3,))] == parent(h)[[1,4,5]] +@test A[OffsetArray([true false; false true], A.offsets)] == [1,4] +@test A[OffsetArray([true true; false true], A.offsets)] == [1,3,4] +@test_throws BoundsError A[[true true; false true]] + # view S = view(A, :, 3) @test S == OffsetArray([1,2], (A.offsets[1],)) @@ -172,9 +183,6 @@ v = view(A0, i1, 1) v = view(A0, 1:1, i1) @test indices(v) === (Base.OneTo(1), -4:-3) -# logical indexing -@test A[A .> 2] == [3,4] - # copy! and fill! a = OffsetArray{Int}((-3:-1,)) fill!(a, -1) @@ -281,7 +289,6 @@ I,J,N = findnz(z) @test I == [-1] @test J == [0] @test N == [2] -h = OffsetArray([-1,1,-2,2,0], (-3,)) @test find(h) == [-2:1;] @test find(x->x>0, h) == [-1,1] @test find(x->x<0, h) == [-2,0] From 109a43fd53953c5e28fc055f9ea46d3877356897 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 15 Aug 2016 22:51:55 -0400 Subject: [PATCH 042/114] fix "improved" correctness of fieldtype the getfield_tfunc was missing a test for whether the fields were all equivalent before concluding that the result type was exact fixes the fix #17953 fix #18037 (cherry picked from commit 06a8d891e71c12a7598b2fa818dcfc29437d5c0b) ref #18045 --- base/inference.jl | 9 +++++---- test/inference.jl | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 869b30aa0f3e2..95471df20d788 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -478,7 +478,7 @@ function getfield_tfunc(s0::ANY, name) # in the current type system typ = limit_type_depth(R, 0, true, filter!(x->isa(x,TypeVar), Any[s.parameters...])) - return typ, isleaftype(s) && typeseq(typ, R) + return typ, isleaftype(s) && isa(R, Type) && typeof(R) === typeof(typ) && typeseq(R, typ) end end end @@ -501,14 +501,15 @@ function getfield_tfunc(s0::ANY, name) elseif length(s.types) == 1 && isempty(s.parameters) return s.types[1], true else - R = reduce(tmerge, Bottom, map(unwrapva,s.types)) #=Union{s.types...}=# + R = reduce(tmerge, Bottom, map(unwrapva, s.types)) #=Union{s.types...}=# + alleq = isa(R, Type) && typeof(R) === typeof(s.types[1]) && typeseq(R, s.types[1]) # do the same limiting as the known-symbol case to preserve type-monotonicity if isempty(s.parameters) - return R, typeseq(R, s.types[1]) + return R, alleq else typ = limit_type_depth(R, 0, true, filter!(x->isa(x,TypeVar), Any[s.parameters...])) - return typ, isleaftype(s) && typeseq(typ, R) + return typ, alleq && isleaftype(s) && typeof(R) === typeof(typ) && typeseq(R, typ) end end end diff --git a/test/inference.jl b/test/inference.jl index f1f4fbb9b1457..a324716b2511b 100644 --- a/test/inference.jl +++ b/test/inference.jl @@ -307,6 +307,21 @@ let T = Array{Tuple{Vararg{Float64,TypeVar(:dim)}},1}, end @test f16530a(:d) == Vector +let T1 = Tuple{Int, Float64}, + T2 = Tuple{Int, Float32}, + T = Tuple{T1, T2} + + global f18037 + f18037() = fieldtype(T, 1) + f18037(i) = fieldtype(T, i) + + @test f18037() === T1 + @test f18037(1) === T1 + @test f18037(2) === T2 + + @test Base.return_types(f18037, ()) == Any[Type{T1}] + @test Base.return_types(f18037, (Int,)) == Any[Type{TypeVar(:T, Tuple{Int, AbstractFloat})}] +end # issue #18015 type Triple18015 From c2a20e10577eacb828f1fb65d28d6ff214a15ed0 Mon Sep 17 00:00:00 2001 From: Pranav Thulasiram Bhat Date: Tue, 16 Aug 2016 22:52:19 +0530 Subject: [PATCH 043/114] Define find and findnz for SparseVector (#18049) (cherry picked from commit bf2c29f1eee66c632d34581b35b7559b298d14ed) --- base/sparse/sparsevector.jl | 51 +++++++++++++++++++++++++++++++++- test/sparsedir/sparsevector.jl | 7 +++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index db2a603232e33..8bcc4c4e32379 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -2,7 +2,7 @@ ### Common definitions -import Base: scalarmax, scalarmin, sort +import Base: scalarmax, scalarmin, sort, find, findnz ### The SparseVector @@ -560,6 +560,55 @@ function getindex{Tv}(A::SparseMatrixCSC{Tv}, I::AbstractVector) SparseVector(n, rowvalB, nzvalB) end +function find{Tv,Ti}(x::SparseVector{Tv,Ti}) + numnz = nnz(x) + I = Array(Ti, numnz) + + nzind = x.nzind + nzval = x.nzval + + count = 1 + @inbounds for i = 1 : numnz + if nzval[i] != 0 + I[count] = nzind[i] + count += 1 + end + end + + count -= 1 + if numnz != count + deleteat!(I, (count+1):numnz) + end + + return I +end + +function findnz{Tv,Ti}(x::SparseVector{Tv,Ti}) + numnz = nnz(x) + + I = Array(Ti, numnz) + V = Array(Tv, numnz) + + nzind = x.nzind + nzval = x.nzval + + count = 1 + @inbounds for i = 1 : numnz + if nzval[i] != 0 + I[count] = nzind[i] + V[count] = nzval[i] + count += 1 + end + end + + count -= 1 + if numnz != count + deleteat!(I, (count+1):numnz) + deleteat!(V, (count+1):numnz) + end + + return (I, V) +end ### Generic functions operating on AbstractSparseVector diff --git a/test/sparsedir/sparsevector.jl b/test/sparsedir/sparsevector.jl index 62a7649cc1e4d..3811cb8bdeb21 100644 --- a/test/sparsedir/sparsevector.jl +++ b/test/sparsedir/sparsevector.jl @@ -225,6 +225,13 @@ let x = SparseVector(10, [2, 7, 9], [2.0, 7.0, 9.0]) @test Base.SparseArrays.dropstored!(x, 5) == SparseVector(10, [7, 9], [7.0, 9.0]) end +# find and findnz tests +@test find(spv_x1) == find(x1_full) +@test findnz(spv_x1) == (find(x1_full), filter(x->x!=0, x1_full)) +let xc = SparseVector(8, [2, 3, 5], [1.25, 0, -0.75]), fc = full(xc) + @test find(xc) == find(fc) + @test findnz(xc) == ([2, 5], [1.25, -0.75]) +end ### Array manipulation From 5562759698c6c3012c5c0e154b86e98e0f360e61 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 20 Aug 2016 00:06:30 -0700 Subject: [PATCH 044/114] Fix indent in sparsevector.jl and remove some unnecessary blank lines [ci skip] (cherry picked from commit 3d373ee29e7d2668eb00299c2837759edbd18afd) --- base/sparse/sparsevector.jl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index 8bcc4c4e32379..95273ad8ad6e0 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -603,8 +603,8 @@ function findnz{Tv,Ti}(x::SparseVector{Tv,Ti}) count -= 1 if numnz != count - deleteat!(I, (count+1):numnz) - deleteat!(V, (count+1):numnz) + deleteat!(I, (count+1):numnz) + deleteat!(V, (count+1):numnz) end return (I, V) @@ -932,7 +932,6 @@ function _binarymap{Tx,Ty}(f::Function, x::AbstractSparseVector{Tx}, y::AbstractSparseVector{Ty}, mode::Int) - 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) R = typeof(f(zero(Tx), zero(Ty))) n = length(x) @@ -1074,7 +1073,6 @@ function _binarymap{Tx,Ty}(f::Function, x::AbstractVector{Tx}, y::AbstractSparseVector{Ty}, mode::Int) - 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) R = typeof(f(zero(Tx), zero(Ty))) n = length(x) @@ -1117,7 +1115,6 @@ function _binarymap{Tx,Ty}(f::Function, x::AbstractSparseVector{Tx}, y::AbstractVector{Ty}, mode::Int) - 0 <= mode <= 2 || throw(ArgumentError("Incorrect mode $mode.")) R = typeof(f(zero(Tx), zero(Ty))) n = length(x) From 7c3adf77fb37e076945a56d4015ac024e348c120 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Tue, 16 Aug 2016 12:06:08 +0100 Subject: [PATCH 045/114] Document known problem with setrounding. Fixes the doc problem mentioned in #17926, but not the underlying problem. (cherry picked from commit f6e9bdf0918624a45a9b165bc4afdc05227efce6) ref #18052 update docs, use new markdown syntax (cherry picked from commit 28eeed85bdcc3b818b53692f07b5ee1d64b87fa4) wrap help text (cherry picked from commit 1bd4c70acf894d4f81eb4157bafd94ffa0a5151c) --- base/rounding.jl | 32 +++++++++++++++++-- .../integers-and-floating-point-numbers.rst | 8 +++-- doc/stdlib/numbers.rst | 28 +++++++++++++++- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/base/rounding.jl b/base/rounding.jl index 43fe9565440e2..cbb740af82889 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -100,12 +100,18 @@ end setrounding(T, mode) Set the rounding mode of floating point type `T`, controlling the rounding of basic -arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), [`/`](:func:`/`) -and [`sqrt`](:func:`sqrt`)) and type conversion. +arithmetic functions ([`+`](:func:`+`), [`-`](:func:`-`), [`*`](:func:`*`), +[`/`](:func:`/`) and [`sqrt`](:func:`sqrt`)) and type conversion. Other numerical +functions may give incorrect or invalid values when using rounding modes other than the +default `RoundNearest`. Note that this may affect other types, for instance changing the rounding mode of `Float64` will change the rounding mode of `Float32`. See [`RoundingMode`](:obj:`RoundingMode`) for available modes. + +!!! warning + + This feature is still experimental, and may give unexpected or incorrect values. """ setrounding(T::Type, mode) @@ -138,6 +144,28 @@ equivalent to: setrounding(T, old) See [`RoundingMode`](:obj:`RoundingMode`) for available rounding modes. + +!!! warning + + This feature is still experimental, and may give unexpected or incorrect values. A + known problem is the interaction with compiler optimisations, e.g. + + julia> setrounding(Float64,RoundDown) do + 1.1 + 0.1 + end + 1.2000000000000002 + + Here the compiler is *constant folding*, that is evaluating a known constant + expression at compile time, however the rounding mode is only changed at runtime, so + this is not reflected in the function result. This can be avoided by moving constants + outside the expression, e.g. + + julia> x = 1.1; y = 0.1; + + julia> setrounding(Float64,RoundDown) do + x + y + end + 1.2 """ function setrounding{T}(f::Function, ::Type{T}, rounding::RoundingMode) old_rounding_raw = rounding_raw(T) diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index 0c6f3cf304f0b..2d72791d49a8e 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -507,12 +507,16 @@ rounded to an appropriate representable value, however, if wanted, the manner in which this rounding is done can be changed according to the rounding modes presented in the `IEEE 754 standard `_:: +.. doctest:: + + + julia> x = 1.1; y = 0.1; - julia> 1.1 + 0.1 + julia> x + y 1.2000000000000002 julia> setrounding(Float64,RoundDown) do - 1.1 + 0.1 + x + y end 1.2 diff --git a/doc/stdlib/numbers.rst b/doc/stdlib/numbers.rst index ae1d47fa34c5f..1d89e90c5db58 100644 --- a/doc/stdlib/numbers.rst +++ b/doc/stdlib/numbers.rst @@ -371,10 +371,14 @@ General Number Functions and Constants .. Docstring generated from Julia source - Set the rounding mode of floating point type ``T``\ , controlling the rounding of basic arithmetic functions (:func:`+`\ , :func:`-`\ , :func:`*`\ , :func:`/` and :func:`sqrt`\ ) and type conversion. + Set the rounding mode of floating point type ``T``\ , controlling the rounding of basic arithmetic functions (:func:`+`\ , :func:`-`\ , :func:`*`\ , :func:`/` and :func:`sqrt`\ ) and type conversion. Other numerical functions may give incorrect or invalid values when using rounding modes other than the default ``RoundNearest``\ . Note that this may affect other types, for instance changing the rounding mode of ``Float64`` will change the rounding mode of ``Float32``\ . See :obj:`RoundingMode` for available modes. + .. warning:: + This feature is still experimental, and may give unexpected or incorrect values. + + .. function:: setrounding(f::Function, T, mode) .. Docstring generated from Julia source @@ -390,6 +394,28 @@ General Number Functions and Constants See :obj:`RoundingMode` for available rounding modes. + .. warning:: + This feature is still experimental, and may give unexpected or incorrect values. A known problem is the interaction with compiler optimisations, e.g. + + .. code-block:: julia + + julia> setrounding(Float64,RoundDown) do + 1.1 + 0.1 + end + 1.2000000000000002 + + Here the compiler is *constant folding*, that is evaluating a known constant expression at compile time, however the rounding mode is only changed at runtime, so this is not reflected in the function result. This can be avoided by moving constants outside the expression, e.g. + + .. code-block:: julia + + julia> x = 1.1; y = 0.1; + + julia> setrounding(Float64,RoundDown) do + x + y + end + 1.2 + + .. function:: get_zero_subnormals() -> Bool .. Docstring generated from Julia source From 27fadb44be87148d1519046d061e880e5f4af479 Mon Sep 17 00:00:00 2001 From: Mus M Date: Tue, 16 Aug 2016 14:35:17 -0400 Subject: [PATCH 046/114] Fix for consistency. (cherry picked from commit 5d2a403c99a815e62b0c368e4e0ddf187cee3243) ref #18044 --- doc/manual/interfaces.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/interfaces.rst b/doc/manual/interfaces.rst index 22b1e808c946f..fc8fbf349f122 100644 --- a/doc/manual/interfaces.rst +++ b/doc/manual/interfaces.rst @@ -66,7 +66,7 @@ A simple example is an iterable sequence of square numbers with a defined length end Base.start(::Squares) = 1 Base.next(S::Squares, state) = (state*state, state+1) - Base.done(S::Squares, s) = s > S.count; + Base.done(S::Squares, state) = state > S.count; Base.eltype(::Type{Squares}) = Int # Note that this is defined for the type Base.length(S::Squares) = S.count; From 1aa11750c693ac9e6e4eedc112f2b9b784b364f5 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 16 Aug 2016 14:49:11 -0500 Subject: [PATCH 047/114] Some indices generalizations for linalg/generic (#18032) (cherry picked from commit f14c2ca47ef74224b6983794e298cd57d71357a7) --- base/linalg/generic.jl | 56 +++++++++++++++++++++--------------------- base/linalg/linalg.jl | 4 ++- test/offsetarray.jl | 4 +++ 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/base/linalg/generic.jl b/base/linalg/generic.jl index 3451999961ba7..261bed72b8c39 100644 --- a/base/linalg/generic.jl +++ b/base/linalg/generic.jl @@ -19,8 +19,8 @@ function generic_scale!(s::Number, X::AbstractArray) end function generic_scale!(C::AbstractArray, X::AbstractArray, s::Number) - if length(C) != length(X) - throw(DimensionMismatch("first array has length $(length(C)) which does not match the length of the second, $(length(X)).")) + if _length(C) != _length(X) + throw(DimensionMismatch("first array has length $(_length(C)) which does not match the length of the second, $(_length(X)).")) end for (IC, IX) in zip(eachindex(C), eachindex(X)) @inbounds C[IC] = X[IX]*s @@ -29,9 +29,9 @@ function generic_scale!(C::AbstractArray, X::AbstractArray, s::Number) end function generic_scale!(C::AbstractArray, s::Number, X::AbstractArray) - if length(C) != length(X) - throw(DimensionMismatch("first array has length $(length(C)) which does not -match the length of the second, $(length(X)).")) + if _length(C) != _length(X) + throw(DimensionMismatch("first array has length $(_length(C)) which does not +match the length of the second, $(_length(X)).")) end for (IC, IX) in zip(eachindex(C), eachindex(X)) @inbounds C[IC] = s*X[IX] @@ -126,7 +126,7 @@ function generic_vecnorm2(x) s = start(x) (v, s) = next(x, s) T = typeof(maxabs) - if isfinite(length(x)*maxabs*maxabs) && maxabs*maxabs != 0 # Scaling not necessary + if isfinite(_length(x)*maxabs*maxabs) && maxabs*maxabs != 0 # Scaling not necessary sum::promote_type(Float64, T) = norm_sqr(v) while !done(x, s) (v, s) = next(x, s) @@ -156,7 +156,7 @@ function generic_vecnormp(x, p) T = typeof(float(norm(v))) end spp::promote_type(Float64, T) = p - if -1 <= p <= 1 || (isfinite(length(x)*maxabs^spp) && maxabs^spp != 0) # scaling not necessary + if -1 <= p <= 1 || (isfinite(_length(x)*maxabs^spp) && maxabs^spp != 0) # scaling not necessary sum::promote_type(Float64, T) = norm(v)^spp while !done(x, s) (v, s) = next(x, s) @@ -253,14 +253,14 @@ end @inline norm(x::Number, p::Real=2) = vecnorm(x, p) function vecdot(x::AbstractArray, y::AbstractArray) - lx = length(x) - if lx != length(y) - throw(DimensionMismatch("first array has length $(lx) which does not match the length of the second, $(length(y)).")) + lx = _length(x) + if lx != _length(y) + throw(DimensionMismatch("first array has length $(lx) which does not match the length of the second, $(_length(y)).")) end if lx == 0 return dot(zero(eltype(x)), zero(eltype(y))) end - s = zero(dot(x[1], y[1])) + s = zero(dot(first(x), first(y))) for (Ix, Iy) in zip(eachindex(x), eachindex(y)) @inbounds s += dot(x[Ix], y[Iy]) end @@ -378,11 +378,11 @@ condskeel(A::AbstractMatrix, x::AbstractVector, p::Real=Inf) = norm(abs(inv(A))* condskeel{T<:Integer}(A::AbstractMatrix{T}, x::AbstractVector, p::Real=Inf) = norm(abs(inv(float(A)))*abs(A)*abs(x), p) function issymmetric(A::AbstractMatrix) - m, n = size(A) - if m != n + indsm, indsn = indices(A) + if indsm != indsn return false end - for i = 1:(n-1), j = (i+1):n + for i = first(indsn):last(indsn)-1, j = (i+1):last(indsn) if A[i,j] != transpose(A[j,i]) return false end @@ -393,11 +393,11 @@ end issymmetric(x::Number) = true function ishermitian(A::AbstractMatrix) - m, n = size(A) - if m != n + indsm, indsn = indices(A) + if indsm != indsn return false end - for i = 1:n, j = i:n + for i = indsn, j = i:last(indsn) if A[i,j] != ctranspose(A[j,i]) return false end @@ -497,26 +497,26 @@ end # BLAS-like in-place y = x*α+y function (see also the version in blas.jl # for BlasFloat Arrays) function axpy!(α, x::AbstractArray, y::AbstractArray) - n = length(x) - if n != length(y) - throw(DimensionMismatch("x has length $n, but y has length $(length(y))")) + n = _length(x) + if n != _length(y) + throw(DimensionMismatch("x has length $n, but y has length $(_length(y))")) end - for i = 1:n - @inbounds y[i] += x[i]*α + for (IY, IX) in zip(eachindex(y), eachindex(x)) + @inbounds y[IY] += x[IX]*α end y end function axpy!{Ti<:Integer,Tj<:Integer}(α, x::AbstractArray, rx::AbstractArray{Ti}, y::AbstractArray, ry::AbstractArray{Tj}) - if length(rx) != length(ry) - throw(DimensionMismatch("rx has length $(length(rx)), but ry has length $(length(ry))")) - elseif minimum(rx) < 1 || maximum(rx) > length(x) + if _length(rx) != _length(ry) + throw(DimensionMismatch("rx has length $(_length(rx)), but ry has length $(_length(ry))")) + elseif !checkindex(Bool, linearindices(x), rx) throw(BoundsError(x, rx)) - elseif minimum(ry) < 1 || maximum(ry) > length(y) + elseif !checkindex(Bool, linearindices(y), ry) throw(BoundsError(y, ry)) end - for i = 1:length(rx) - @inbounds y[ry[i]] += x[rx[i]]*α + for (IY, IX) in zip(eachindex(ry), eachindex(rx)) + @inbounds y[ry[IY]] += x[rx[IX]]*α end y end diff --git a/base/linalg/linalg.jl b/base/linalg/linalg.jl index dc36d649fae31..556020776082a 100644 --- a/base/linalg/linalg.jl +++ b/base/linalg/linalg.jl @@ -10,7 +10,9 @@ import Base: USE_BLAS64, abs, big, ceil, conj, convert, copy, copy!, copy_transp imag, inv, isapprox, kron, ndims, parent, power_by_squaring, print_matrix, promote_rule, real, round, setindex!, show, similar, size, transpose, transpose!, trunc -using Base: promote_op +using Base: promote_op, _length +# We use `_length` because of non-1 indices; releases after julia 0.5 +# can go back to `length`. `_length(A)` is equivalent to `length(linearindices(A))`. export # Modules diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 2bb5104a88103..4f4e70933a55d 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -294,6 +294,10 @@ I,J,N = findnz(z) @test find(x->x<0, h) == [-2,0] @test find(x->x==0, h) == [2] +@test_approx_eq vecnorm(v) vecnorm(parent(v)) +@test_approx_eq vecnorm(A) vecnorm(parent(A)) +@test_approx_eq vecdot(v, v) vecdot(v0, v0) + v = OffsetArray([1,1e100,1,-1e100], (-3,))*1000 v2 = OffsetArray([1,-1e100,1,1e100], (5,))*1000 @test isa(v, OffsetArray) From 42a32623b64362f89c82e165987ccb8527c213f1 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 16 Aug 2016 13:39:59 -0400 Subject: [PATCH 048/114] fix #18051, allocation in sparse vector getindex Caused by the return type of `ord()` being unknown. (cherry picked from commit b74bc6f38519fbce5ee5359a1f2cdac70b424fc6) ref #18060 --- base/sort.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/sort.jl b/base/sort.jl index d388e55f6647c..91e752202c57b 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -184,6 +184,7 @@ for s in [:searchsortedfirst, :searchsortedlast, :searchsorted] $s(v::AbstractVector, x; lt=isless, by=identity, rev::Bool=false, order::Ordering=Forward) = $s(v,x,ord(lt,by,rev,order)) + $s(v::AbstractVector, x) = $s(v, x, Forward) end end From bc7f384cb3e465ec80014d154b1c70dae6a61885 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 11 Aug 2016 04:52:25 -0700 Subject: [PATCH 049/114] Add cygpath_w call so `make -C test/perf` works in cygwin-cross (cherry picked from commit 03f37f292bd6b239f6a8c319a9f3474afcce150f) ref #18022 --- test/perf/Makefile | 6 +----- test/perf/shootout/fasta.jl | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/perf/Makefile b/test/perf/Makefile index 0ff723aa70f7b..adc07a2425e64 100644 --- a/test/perf/Makefile +++ b/test/perf/Makefile @@ -9,11 +9,7 @@ all: micro kernel cat shootout blas lapack simd sort spell sparse micro kernel cat shootout blas lapack simd sort spell sparse: @$(MAKE) $(QUIET_MAKE) -C $(SRCDIR)/shootout -ifneq ($(OS),WINNT) - @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/$@/perf.jl | perl -nle '@_=split/,/; printf "%-18s %8.3f %8.3f %8.3f %8.3f\n", $$_[1], $$_[2], $$_[3], $$_[4], $$_[5]' -else - @$(call spawn,$(JULIA_EXECUTABLE)) $(SRCDIR)/$@/perf.jl 2> /dev/null -endif + @$(call spawn,$(JULIA_EXECUTABLE)) $(call cygpath_w,$(SRCDIR)/$@/perf.jl) | perl -nle '@_=split/,/; printf "%-18s %8.3f %8.3f %8.3f %8.3f\n", $$_[1], $$_[2], $$_[3], $$_[4], $$_[5]' codespeed: @$(MAKE) $(QUIET_MAKE) -C $(SRCDIR)/shootout diff --git a/test/perf/shootout/fasta.jl b/test/perf/shootout/fasta.jl index 13bbec2c22875..3746a15e7e8ae 100644 --- a/test/perf/shootout/fasta.jl +++ b/test/perf/shootout/fasta.jl @@ -55,7 +55,7 @@ end rng_state = 42.0 function fasta(n=25000000) - repeat_fasta(alu, 2n) - random_fasta(iub1, iub2, 3n) - random_fasta(homosapiens1, homosapiens2, 5n) + repeat_fasta(alu, 2n) + random_fasta(iub1, iub2, 3n) + random_fasta(homosapiens1, homosapiens2, 5n) end From 0388e37bb462a4b0ddf1e3e34ba930c2b39181ae Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 11 Aug 2016 11:38:24 -0700 Subject: [PATCH 050/114] fix deprecated syntax "global rng_state::Float64" (cherry picked from commit 2359ef8bbc26ffa0d02e5eb54280a82f8571692b) ref #18022 --- test/perf/shootout/fasta.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/shootout/fasta.jl b/test/perf/shootout/fasta.jl index 3746a15e7e8ae..fe8129d1d74c7 100644 --- a/test/perf/shootout/fasta.jl +++ b/test/perf/shootout/fasta.jl @@ -22,7 +22,7 @@ const IA = 3877.0 const IC = 29573.0 function gen_random() - global rng_state::Float64 = ((rng_state::Float64 * IA + IC) % IM) / IM + global rng_state = ((rng_state::Float64 * IA + IC) % IM) / IM end function repeat_fasta(src, n) k = length(src) From a3e144e57f3f7a378585861cc4f0016813f5725d Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 13 Aug 2016 21:24:45 -0700 Subject: [PATCH 051/114] Fix a row-indexing bug with sparse matrices that have non-Int indices, introduced by #13612 and which has caused `make -C test/perf` to fail for the last 9 months searchsortedfirst does not have methods for general Integer indices (cherry picked from commit b1d53218ac9c4721988d0f8dbf3ce25085a9d026) ref #18022 --- base/sparse/sparsevector.jl | 4 ++-- test/sparsedir/sparse.jl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/base/sparse/sparsevector.jl b/base/sparse/sparsevector.jl index 95273ad8ad6e0..26076fcce833c 100644 --- a/base/sparse/sparsevector.jl +++ b/base/sparse/sparsevector.jl @@ -426,8 +426,8 @@ function Base.getindex{Tv,Ti}(A::SparseMatrixCSC{Tv,Ti}, i::Integer, J::Abstract @inbounds for j = 1:nJ col = J[j] rowI = i - ptrA = colptrA[col] - stopA = colptrA[col+1]-1 + ptrA = Int(colptrA[col]) + stopA = Int(colptrA[col+1]-1) if ptrA <= stopA if rowvalA[ptrA] <= rowI ptrA = searchsortedfirst(rowvalA, rowI, ptrA, stopA, Base.Order.Forward) diff --git a/test/sparsedir/sparse.jl b/test/sparsedir/sparse.jl index e6d4e9767e2b1..4cfc2b248097d 100644 --- a/test/sparsedir/sparse.jl +++ b/test/sparsedir/sparse.jl @@ -1588,3 +1588,8 @@ end # Test temporary fix for issue #16548 in PR #16979. Brittle. Expect to remove with `\` revisions. @test which(\, (SparseMatrixCSC, AbstractVecOrMat)).module == Base.SparseArrays + +# Row indexing a SparseMatrixCSC with non-Int integer type +let A = sparse(UInt32[1,2,3], UInt32[1,2,3], [1.0,2.0,3.0]) + @test A[1,1:3] == A[1,:] == [1,0,0] +end From 96ef8f23430f1683b77518a4c37b37b08ff7abfd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 11 Aug 2016 01:01:21 -0700 Subject: [PATCH 052/114] add support for split debuginfo files There seem to be a few variations on the tools and arguments available for creating split debuginfo files. I used the following: objcopy --only-keep-debug libjulia.so libjulia.so.debug strip -g libjulia.so objcopy --add-gnu-debuglink=libjulia.so.debug libjulia.so (cherry picked from commit b67fd06f11cc55f0c7823ed606bced2e2e84d64c) ref #17962 --- src/debuginfo.cpp | 553 +++++++++++++++++++++++++++++----------------- 1 file changed, 352 insertions(+), 201 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index b61abe00e817c..c02eda00de75b 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -748,6 +748,8 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, #ifdef _OS_DARWIN_ #include +#else +#define LC_UUID 0 #endif #ifndef _OS_WINDOWS_ #include @@ -761,11 +763,10 @@ typedef struct { typedef std::map obfiletype; static obfiletype objfilemap; -#ifdef _OS_DARWIN_ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) { # ifdef LLVM37 - for (auto Load : obj->load_commands ()) { + for (auto Load : obj->load_commands()) # else # ifdef LLVM35 uint32_t LoadCommandCount = obj->getHeader().ncmds; @@ -773,8 +774,9 @@ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) uint32_t LoadCommandCount = obj->getHeader().NumLoadCommands; # endif llvm::object::MachOObjectFile::LoadCommandInfo Load = obj->getFirstLoadCommandInfo(); - for (unsigned I = 0; ; ++I) { + for (unsigned I = 0; ; ++I) # endif + { if ( # ifdef LLVM35 Load.C.cmd == LC_UUID @@ -782,7 +784,7 @@ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) Load.C.Type == LC_UUID # endif ) { - memcpy(uuid,((MachO::uuid_command*)Load.Ptr)->uuid,16); + memcpy(uuid, ((const MachO::uuid_command*)Load.Ptr)->uuid, 16); return true; } # ifndef LLVM37 @@ -796,6 +798,115 @@ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) } return false; } + +#ifdef LLVM36 +struct debug_link_info { + StringRef filename; + uint32_t crc32; +}; +static debug_link_info getDebuglink(const object::ObjectFile &Obj) +{ + debug_link_info info = {}; + for (const object::SectionRef &Section: Obj.sections()) { + StringRef sName; + if (!Section.getName(sName) && sName == ".gnu_debuglink") { + StringRef Contents; + if (!Section.getContents(Contents)) { + size_t length = Contents.find('\0'); + info.filename = Contents.substr(0, length); + info.crc32 = *(const uint32_t*)Contents.substr(LLT_ALIGN(length + 1, 4), 4).data(); + break; + } + } + } + return info; +} +/* + * crc function from http://svnweb.freebsd.org/base/head/sys/libkern/crc32.c (and lldb) + * + * COPYRIGHT (C) 1986 Gary S. Brown. You may use this program, or + * code or tables extracted from it, as desired without restriction. + */ +static uint32_t +calc_gnu_debuglink_crc32(const void *buf, size_t size) +{ + static const uint32_t g_crc32_tab[] = + { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, + 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, + 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, + 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, + 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, + 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, + 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, + 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, + 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, + 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, + 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, + 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d + }; + const uint8_t *p = (const uint8_t *)buf; + uint32_t crc; + + crc = ~0U; + while (size--) + crc = g_crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); + return crc ^ ~0U; +} +static ErrorOr> openDebugInfo(StringRef debuginfopath, const debug_link_info &info) +{ + auto SplitFile = MemoryBuffer::getFile(debuginfopath); + if (std::error_code EC = SplitFile.getError()) + return EC; + + uint32_t crc32 = calc_gnu_debuglink_crc32( + SplitFile.get()->getBufferStart(), + SplitFile.get()->getBufferSize()); + if (crc32 != info.crc32) + return object::object_error::arch_not_found; + + auto error_splitobj = object::ObjectFile::createObjectFile( + SplitFile.get().get()->getMemBufferRef(), + sys::fs::file_magic::unknown); + if (std::error_code EC = error_splitobj.getError()) + return EC; + + // successfully validated and loaded split debug info file + return object::OwningBinary( + std::move(error_splitobj.get()), + std::move(SplitFile.get())); +} #endif static uint64_t jl_sysimage_base; @@ -817,55 +928,57 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, *context = NULL; *slide = 0; *section_slide = 0; + // GOAL: Determine containing Library -// Assigning fname, fbase, msize +// Assigning fname, fbase #ifdef _OS_WINDOWS_ IMAGEHLP_MODULE64 ModuleInfo; - bool isvalid; ModuleInfo.SizeOfStruct = sizeof(IMAGEHLP_MODULE64); jl_in_stackwalk = 1; - isvalid = SymGetModuleInfo64(GetCurrentProcess(), (DWORD64)pointer, &ModuleInfo); + bool isvalid = SymGetModuleInfo64(GetCurrentProcess(), (DWORD64)pointer, &ModuleInfo); jl_in_stackwalk = 0; - if (isvalid) { - char *fname = ModuleInfo.LoadedImageName; - if (!fname[0]) // empirically, LoadedImageName might be missing - fname = ModuleInfo.ImageName; - DWORD64 fbase = ModuleInfo.BaseOfImage; - bool insysimage = (fbase == jl_sysimage_base); - if (isSysImg) - *isSysImg = insysimage; - if (onlySysImg && !insysimage) { - return false; - } - static char frame_info_func[ - sizeof(SYMBOL_INFO) + - MAX_SYM_NAME * sizeof(TCHAR)]; - DWORD64 dwDisplacement64 = 0; - DWORD64 dwAddress = pointer; - PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)frame_info_func; - pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); - pSymbol->MaxNameLen = MAX_SYM_NAME; - jl_in_stackwalk = 1; - if (SymFromAddr(GetCurrentProcess(), dwAddress, &dwDisplacement64, - pSymbol)) { - // SymFromAddr returned success - // errors are ignored, but are hopefully patched up by - // using llvm to read the object (below) - if (name) - jl_copy_str(name, pSymbol->Name); - if (saddr) - *saddr = (void*)(uintptr_t)pSymbol->Address; - } - else if (saddr) { - *saddr = NULL; - } + if (!isvalid) return false; + + StringRef fname = ModuleInfo.LoadedImageName; + if (fname.empty()) // empirically, LoadedImageName might be missing + fname = ModuleInfo.ImageName; + DWORD64 fbase = ModuleInfo.BaseOfImage; + bool insysimage = (fbase == jl_sysimage_base); + if (isSysImg) + *isSysImg = insysimage; + if (onlySysImg && !insysimage) { + return false; + } + static char frame_info_func[ + sizeof(SYMBOL_INFO) + + MAX_SYM_NAME * sizeof(TCHAR)]; + DWORD64 dwDisplacement64 = 0; + DWORD64 dwAddress = pointer; + PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)frame_info_func; + pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); + pSymbol->MaxNameLen = MAX_SYM_NAME; + jl_in_stackwalk = 1; + if (SymFromAddr(GetCurrentProcess(), dwAddress, &dwDisplacement64, + pSymbol)) { + // SymFromAddr returned success + // errors are ignored, but are hopefully patched up by + // using llvm to read the object (below) + if (name) + jl_copy_str(name, pSymbol->Name); + if (saddr) + *saddr = (void*)(uintptr_t)pSymbol->Address; + } + else if (saddr) { + *saddr = NULL; + } - // If we didn't find the filename before in the debug - // info, use the dll name - if (filename && !*filename) - jl_copy_str(filename, fname); + // If we didn't find the filename before in the debug + // info, use the dll name + if (filename && !*filename) + jl_copy_str(filename, fname.data()); + + jl_in_stackwalk = 0; - jl_in_stackwalk = 0; #else // ifdef _OS_WINDOWS_ Dl_info dlinfo; int dladdr_success; @@ -876,201 +989,239 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, #else dladdr_success = dladdr((void*)pointer, &dlinfo) != 0; #endif + if (!dladdr_success || !dlinfo.dli_fname) + return false; - if (dladdr_success && dlinfo.dli_fname) { #ifdef __GLIBC__ - // dlinfo.dli_fbase is not the right value for the main executable on linux - fbase = (uintptr_t)extra_info->l_addr; + // dlinfo.dli_fbase is not the right value for the main executable on linux + fbase = (uintptr_t)extra_info->l_addr; #else - fbase = (uintptr_t)dlinfo.dli_fbase; -#endif - const char *fname; - if (saddr) - *saddr = dlinfo.dli_saddr; + fbase = (uintptr_t)dlinfo.dli_fbase; +#endif + StringRef fname; + if (saddr) + *saddr = dlinfo.dli_saddr; + bool insysimage = (fbase == jl_sysimage_base); + if (isSysImg) + *isSysImg = insysimage; + if (onlySysImg && !insysimage) { + return false; + } + // In case we fail with the debug info lookup, we at least still + // have the function name, even if we don't have line numbers + if (name) + jl_copy_str(name, dlinfo.dli_sname); + if (filename) + jl_copy_str(filename, dlinfo.dli_fname); + fname = dlinfo.dli_fname; +#endif // ifdef _OS_WINDOWS_ + + int isdarwin = 0, islinux = 0, iswindows = 0; #if defined(_OS_DARWIN_) - size_t msize = (size_t)(((uint64_t)-1)-fbase); + isdarwin = 1; +#elif defined(_OS_LINUX_) || defined(_OS_FREEBSD_) + islinux = 1; +#elif defined(_OS_WINDOWS_) + iswindows = 1; +#endif + +#ifndef LLVM35 + if (iswindows) { + return true; + } #endif - bool insysimage = (fbase == jl_sysimage_base); - if (isSysImg) - *isSysImg = insysimage; - if (onlySysImg && !insysimage) { - return false; - } - // In case we fail with the debug info lookup, we at least still - // have the function name, even if we don't have line numbers - if (name) - jl_copy_str(name, dlinfo.dli_sname); - if (filename) - jl_copy_str(filename, dlinfo.dli_fname); - fname = dlinfo.dli_fname; -#endif // ifdef _OS_WINDOWS_ // GOAL: Read debuginfo from file -#if !defined(_OS_WINDOWS_) || defined(LLVM35) - // TODO: need read/write lock here for objfilemap synchronization - obfiletype::iterator it = objfilemap.find(fbase); - if (it != objfilemap.end()) { - // Return cached value - *obj = it->second.obj; - *context = it->second.ctx; - *slide = it->second.slide; - *section_slide = it->second.section_slide; - } - else { + // TODO: need read/write lock here for objfilemap synchronization + obfiletype::iterator it = objfilemap.find(fbase); + if (it != objfilemap.end()) { + // Return cached value + *obj = it->second.obj; + *context = it->second.ctx; + *slide = it->second.slide; + *section_slide = it->second.section_slide; + return true; + } + // GOAL: Assign errorobj -#if defined(_OS_DARWIN_) + StringRef objpath; + std::string debuginfopath; + uint8_t uuid[16], uuid2[16]; + if (isdarwin) { + size_t msize = (size_t)(((uint64_t)-1) - fbase); #ifdef LLVM36 - std::unique_ptr membuf = MemoryBuffer::getMemBuffer( - StringRef((const char *)fbase, msize), "", false); - auto origerrorobj = llvm::object::ObjectFile::createObjectFile( - membuf->getMemBufferRef(), sys::fs::file_magic::unknown); -#elif defined(LLVM35) - MemoryBuffer *membuf = MemoryBuffer::getMemBuffer( + std::unique_ptr membuf = MemoryBuffer::getMemBuffer( StringRef((const char *)fbase, msize), "", false); - std::unique_ptr buf(membuf); - auto origerrorobj = llvm::object::ObjectFile::createObjectFile( - buf, sys::fs::file_magic::unknown); -#else - MemoryBuffer *membuf = MemoryBuffer::getMemBuffer( - StringRef((const char *)fbase, msize), "", false); - llvm::object::ObjectFile *origerrorobj = llvm::object::ObjectFile::createObjectFile( - membuf); -#endif - if (!origerrorobj) { - objfileentry_t entry = {*obj,*context,*slide,*section_slide}; - objfilemap[fbase] = entry; - return true; - } - -#ifdef LLVM36 - *obj = (llvm::object::MachOObjectFile *)origerrorobj.get().release(); + auto origerrorobj = llvm::object::ObjectFile::createObjectFile( + membuf->getMemBufferRef(), sys::fs::file_magic::unknown); #elif defined(LLVM35) - *obj = (llvm::object::MachOObjectFile *)origerrorobj.get(); + MemoryBuffer *membuf = MemoryBuffer::getMemBuffer( + StringRef((const char *)fbase, msize), "", false); + std::unique_ptr buf(membuf); + auto origerrorobj = llvm::object::ObjectFile::createObjectFile( + buf, sys::fs::file_magic::unknown); #else - *obj = (llvm::object::MachOObjectFile *)origerrorobj; + MemoryBuffer *membuf = MemoryBuffer::getMemBuffer( + StringRef((const char *)fbase, msize), "", false); + std::unique_ptr origerrorobj(llvm::object::ObjectFile::createObjectFile( + membuf)); #endif - llvm::object::MachOObjectFile *morigobj = (llvm::object::MachOObjectFile *)*obj; - - - // First find the uuid of the object file (we'll use this to make sure we find the - // correct debug symbol file). - uint8_t uuid[16], uuid2[16]; - if (!getObjUUID(morigobj,uuid)) { - objfileentry_t entry = {*obj,*context,*slide,*section_slide}; - objfilemap[fbase] = entry; - return true; - } + if (!origerrorobj) { + objfileentry_t entry = {}; + objfilemap[fbase] = entry; + return true; + } - // On OS X debug symbols are not contained in the dynamic library and that's why - // we can't have nice things (easily). For now we only support .dSYM files in the same directory - // as the shared library. In the future we may use DBGCopyFullDSYMURLForUUID from CoreFoundation to make - // use of spotlight to find the .dSYM file. - char dsympath[PATH_MAX]; - strlcpy(dsympath, fname, sizeof(dsympath)); - strlcat(dsympath, ".dSYM/Contents/Resources/DWARF/", sizeof(dsympath)); - strlcat(dsympath, strrchr(fname,'/')+1, sizeof(dsympath)); -#ifdef LLVM35 - auto errorobj = llvm::object::ObjectFile::createObjectFile(dsympath); + llvm::object::MachOObjectFile *morigobj = (llvm::object::MachOObjectFile*) +#ifdef LLVM36 + origerrorobj.get().get(); #else - llvm::object::ObjectFile *errorobj = llvm::object::ObjectFile::createObjectFile(dsympath); + origerrorobj.get(); #endif -#else // ifndef _OS_DARWIN_ + // First find the uuid of the object file (we'll use this to make sure we find the + // correct debug symbol file). + if (!getObjUUID(morigobj, uuid)) { + objfileentry_t entry = {}; + objfilemap[fbase] = entry; + return true; + } - // On Linux systems we need to mmap another copy because of the permissions on the mmap'ed shared library. - // On Windows we need to mmap another copy since reading the in-memory copy seems to return object_error:unexpected_eof + // On OS X debug symbols are not contained in the dynamic library. + // For now we only support .dSYM files in the same directory + // as the shared library. In the future we may use DBGCopyFullDSYMURLForUUID from CoreFoundation to make + // use of spotlight to find the .dSYM file. + size_t sep = fname.rfind('/'); + debuginfopath = fname; + debuginfopath += ".dSYM/Contents/Resources/DWARF/"; + debuginfopath += fname.substr(sep + 1); + objpath = debuginfopath; + } + else { + // On Linux systems we need to mmap another copy because of the permissions on the mmap'ed shared library. + // On Windows we need to mmap another copy since reading the in-memory copy seems to return object_error:unexpected_eof + objpath = fname; + } #ifdef LLVM35 - auto errorobj = llvm::object::ObjectFile::createObjectFile(fname); + auto errorobj = llvm::object::ObjectFile::createObjectFile(objpath); #else - llvm::object::ObjectFile *errorobj = llvm::object::ObjectFile::createObjectFile(fname); + std::unique_ptr errorobj(llvm::object::ObjectFile::createObjectFile(objpath)); #endif -#endif // ifdef _OS_DARWIN_ // GOAL: Assign *obj, *context, *slide (if above succeeded) - if (errorobj) { + if (errorobj) { #ifdef LLVM36 - auto binary = errorobj.get().takeBinary(); - *obj = binary.first.release(); - binary.second.release(); -#elif defined(LLVM35) - *obj = errorobj.get(); + auto *debugobj = errorobj->getBinary(); #else - *obj = errorobj; + auto *debugobj = errorobj.get(); #endif -#ifdef _OS_DARWIN_ - if (getObjUUID((llvm::object::MachOObjectFile *)*obj,uuid2) && - memcmp(uuid,uuid2,sizeof(uuid)) == 0) { -#endif -#ifdef LLVM37 - *context = new DWARFContextInMemory(**obj); -#elif defined(LLVM36) - *context = DIContext::getDWARFContext(**obj); -#else - *context = DIContext::getDWARFContext(const_cast(*obj)); -#endif - *slide = -(int64_t)fbase; -#ifdef _OS_DARWIN_ - } - else { - // If we're here the, the dsym does not match the dylib. Use the original - // object instead. For consistency (and to make sure we get a sensible size - // for the memory buffer), we also use a fresh copy mapped from - // the file rather than reusing the one in memory. We may want to revisit - // that in the future (ideally, once we support fewer LLVM versions). - errorobj = llvm::object::ObjectFile::createObjectFile(fname); - assert(errorobj); + + if (islinux) { #ifdef LLVM36 - auto binary = errorobj.get().takeBinary(); - *obj = binary.first.release(); - binary.second.release(); -#elif defined(LLVM35) - *obj = errorobj.get(); -#else - *obj = errorobj; -#endif - delete morigobj; + // if the file has a .gnu_debuglink section, + // try to load its companion file instead + // in the expected locations + // for now, we don't support the build-id method + debug_link_info info = getDebuglink(*debugobj); + if (!info.filename.empty()) { + size_t sep = fname.rfind('/'); + ErrorOr> DebugInfo(std::errc::no_such_file_or_directory); + if (fname.substr(sep + 1) != info.filename) { + debuginfopath = fname.substr(0, sep + 1); + debuginfopath += info.filename; + DebugInfo = openDebugInfo(debuginfopath, info); } -#endif -#if defined(_OS_WINDOWS_) - assert((*obj)->isCOFF()); - const llvm::object::COFFObjectFile *coffobj = (const llvm::object::COFFObjectFile *)*obj; - const llvm::object::pe32plus_header *pe32plus; - coffobj->getPE32PlusHeader(pe32plus); - if (pe32plus != NULL) { - *slide = pe32plus->ImageBase - fbase; - *section_slide = -(int64_t)pe32plus->ImageBase; + if (DebugInfo.getError()) { + debuginfopath = fname.substr(0, sep + 1); + debuginfopath += ".debug/"; + debuginfopath += info.filename; + DebugInfo = openDebugInfo(debuginfopath, info); + } + if (DebugInfo.getError()) { + debuginfopath = "/usr/lib/debug/"; + debuginfopath += fname.substr(0, sep + 1); + debuginfopath += info.filename; + DebugInfo = openDebugInfo(debuginfopath, info); + } + if (DebugInfo.getError()) { + // no split debug info found + // should we warn the user? } else { - const llvm::object::pe32_header *pe32; - coffobj->getPE32Header(pe32); - if (pe32 == NULL) { - *obj = NULL; - *context = NULL; - *slide = 0; - } - else { - *slide = pe32->ImageBase - fbase; - *section_slide = -(int64_t)pe32->ImageBase; - } + errorobj = std::move(DebugInfo); + debugobj = errorobj->getBinary(); } + } #endif + } + + if (isdarwin) { + // verify the UUID matches + if (!getObjUUID((llvm::object::MachOObjectFile*)debugobj, uuid2) || + memcmp(uuid, uuid2, sizeof(uuid)) != 0) { + objfileentry_t entry = {}; + objfilemap[fbase] = entry; + return true; + } + } + + if (iswindows) { +#ifdef LLVM35 + assert(debugobj->isCOFF()); + const llvm::object::COFFObjectFile *coffobj = (const llvm::object::COFFObjectFile*)debugobj; + const llvm::object::pe32plus_header *pe32plus; + coffobj->getPE32PlusHeader(pe32plus); + if (pe32plus != NULL) { + *slide = pe32plus->ImageBase - fbase; + *section_slide = -(int64_t)pe32plus->ImageBase; } -#ifdef LLVM39 else { - // TODO: report the error instead of silently consuming it? - // jl_error might run into the same error again... - consumeError(errorobj.takeError()); + const llvm::object::pe32_header *pe32; + coffobj->getPE32Header(pe32); + if (pe32 == NULL) { + objfileentry_t entry = {}; + objfilemap[fbase] = entry; + return true; + } + else { + *slide = pe32->ImageBase - fbase; + *section_slide = -(int64_t)pe32->ImageBase; + } } #endif - - // update cache - objfileentry_t entry = {*obj,*context,*slide,*section_slide}; - objfilemap[fbase] = entry; } + else { + *slide = -(int64_t)fbase; + } + +#ifdef LLVM37 + *context = new DWARFContextInMemory(*debugobj); +#elif defined(LLVM36) + *context = DIContext::getDWARFContext(*debugobj); +#else + *context = DIContext::getDWARFContext(debugobj); +#endif + *obj = debugobj; +#ifdef LLVM36 + auto binary = errorobj->takeBinary(); + binary.first.release(); + binary.second.release(); +#else + errorobj.release(); #endif - return true; } - return false; +#ifdef LLVM39 + else { + // TODO: report the error instead of silently consuming it? + // jl_error might run into the same error again... + consumeError(errorobj.takeError()); + } +#endif + + // update cache + objfileentry_t entry = {*obj, *context, *slide, *section_slide}; + objfilemap[fbase] = entry; + return true; } // *name and *filename should be either NULL or malloc'd pointer From e5f7f5cf344968bc17378e27d212a9d405fe7ced Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 15 Aug 2016 13:00:55 -0700 Subject: [PATCH 053/114] test libccalltest.so with split debug info (cherry picked from commit 4083d9b9f918d31744d92d2e660d9b8a46980bf8) ref #17962 --- Make.inc | 1 + base/stacktraces.jl | 4 ++-- src/Makefile | 23 ++++++++++++++++++++++- src/debuginfo.cpp | 22 ++++++++++++++++++---- src/stackwalk.c | 10 +++++++--- test/stacktraces.jl | 10 ++++++++++ 6 files changed, 60 insertions(+), 10 deletions(-) diff --git a/Make.inc b/Make.inc index 568a94cb22cf7..003f50db78376 100644 --- a/Make.inc +++ b/Make.inc @@ -460,6 +460,7 @@ endif #ARCH LD := link endif #USEMSVC RANLIB := $(CROSS_COMPILE)ranlib +OBJCOPY := $(CROSS_COMPILE)objcopy # file extensions ifeq ($(OS), WINNT) diff --git a/base/stacktraces.jl b/base/stacktraces.jl index b4be771e44628..55a7d901225ad 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -56,7 +56,7 @@ immutable StackFrame # this type should be kept platform-agnostic so that profil "true if the code is from an inlined frame" inlined::Bool "representation of the pointer to the execution context as returned by `backtrace`" - pointer::Int64 # Large enough to be read losslessly on 32- and 64-bit machines. + pointer::UInt64 # Large enough to be read losslessly on 32- and 64-bit machines. end StackFrame(func, file, line) = StackFrame(func, file, line, Nullable{LambdaInfo}(), false, false, 0) @@ -123,7 +123,7 @@ inlined at that point, innermost function first. """ function lookup(pointer::Ptr{Void}) infos = ccall(:jl_lookup_code_address, Any, (Ptr{Void}, Cint), pointer - 1, false) - isempty(infos) && return [StackFrame(empty_sym, empty_sym, -1, Nullable{LambdaInfo}(), true, false, convert(Int64, pointer))] + isempty(infos) && return [StackFrame(empty_sym, empty_sym, -1, Nullable{LambdaInfo}(), true, false, convert(UInt64, pointer))] res = Array{StackFrame}(length(infos)) for i in 1:length(infos) info = infos[i] diff --git a/src/Makefile b/src/Makefile index ebf8219150674..d7ec864f27e42 100644 --- a/src/Makefile +++ b/src/Makefile @@ -123,8 +123,26 @@ $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.cpp $(SRCDIR)/llvm-version.h $(HEADERS) $(LLV @$(call PRINT_CC, $(CXX) $(shell $(LLVM_CONFIG_HOST) --cxxflags) $(CPPFLAGS) $(CXXFLAGS) $(DEBUGFLAGS) -c $< -o $@) libccalltest: $(build_shlibdir)/libccalltest.$(SHLIB_EXT) + +ifeq ($(OS), Linux) +JULIA_SPLITDEBUG := 1 +else +JULIA_SPLITDEBUG := 0 +endif $(build_shlibdir)/libccalltest.$(SHLIB_EXT): $(SRCDIR)/ccalltest.c - @$(call PRINT_CC, $(CC) $(CFLAGS) $(CPPFLAGS) $(DEBUGFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS)) + @$(call PRINT_CC, $(CC) $(CFLAGS) $(CPPFLAGS) $(DEBUGFLAGS) -O3 $< $(fPIC) -shared -o $@.tmp $(LDFLAGS)) +ifeq ($(JULIA_SPLITDEBUG),1) + @# Create split debug info file for libccalltest stacktraces test + @# packagers should disable this by setting JULIA_SPLITDEBUG=0 if this is already done by your build system + $(OBJCOPY) --only-keep-debug $@.tmp $@.debug + $(OBJCOPY) --strip-debug $@.tmp + $(OBJCOPY) --add-gnu-debuglink=$@.debug $@.tmp +endif + @## clang should have made the dSYM split-debug directory, + @## but we are intentionally not going to give it the correct name + @## because we want to test the non-default debug configuration + @#rm -r $@.dSYM && mv $@.tmp.dSYM $@.dSYM + mv $@.tmp $@ julia_flisp.boot.inc.phony: $(BUILDDIR)/julia_flisp.boot.inc @@ -220,9 +238,11 @@ ifneq ($(OS), WINNT) @ln -sf libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia-debug.$(SHLIB_EXT) endif $(DSYMUTIL) $@ + $(BUILDDIR)/libjulia-debug.a: $(SRCDIR)/julia.expmap $(DOBJS) $(BUILDDIR)/flisp/libflisp-debug.a $(BUILDDIR)/support/libsupport-debug.a rm -f $@ @$(call PRINT_LINK, ar -rcs $@ $(DOBJS)) + libjulia-debug: $(build_shlibdir)/libjulia-debug.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT): $(SRCDIR)/julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a $(LIBUV) @@ -233,6 +253,7 @@ ifneq ($(OS), WINNT) @ln -sf libjulia.$(JL_MAJOR_MINOR_SHLIB_EXT) $(build_shlibdir)/libjulia.$(SHLIB_EXT) endif $(DSYMUTIL) $@ + $(BUILDDIR)/libjulia.a: julia.expmap $(OBJS) $(BUILDDIR)/flisp/libflisp.a $(BUILDDIR)/support/libsupport.a rm -f $@ @$(call PRINT_LINK, ar -rcs $@ $(OBJS)) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index c02eda00de75b..c7e598fc5de4e 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -666,10 +666,19 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, // This function is not allowed to reference any TLS variables // since it can be called from an unmanaged thread on OSX. if (!context) { - if (demangle && (*frames)[0].func_name != NULL) { - char *oldname = (*frames)[0].func_name; - (*frames)[0].func_name = jl_demangle(oldname); - free(oldname); + if (demangle) { + if ((*frames)[0].func_name != NULL) { + char *oldname = (*frames)[0].func_name; + (*frames)[0].func_name = jl_demangle(oldname); + free(oldname); + } + else { + // We do this to hide the jlcall wrappers when getting julia backtraces, + // but it is still good to have them for regular lookup of C frames. + // Technically not true, but we don't want them + // in julia backtraces, so close enough + (*frames)[0].fromC = 1; + } } return 1; } @@ -686,6 +695,9 @@ static int lookup_pointer(DIContext *context, jl_frame_t **frames, int fromC = (*frames)[0].fromC; int n_frames = inlineInfo.getNumberOfFrames(); + if (n_frames == 0) + // no line number info available in the context, return without the context + return lookup_pointer(NULL, frames, pointer, demangle, noInline); if (noInline) n_frames = 1; if (n_frames > 1) { @@ -921,6 +933,7 @@ extern "C" void jl_register_fptrs(uint64_t sysimage_base, void **fptrs, jl_lambd sysimg_fvars_n = n; } +extern "C" void jl_refresh_dbg_module_list(void); bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, llvm::DIContext **context, int64_t *slide, int64_t *section_slide, bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) { @@ -934,6 +947,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, #ifdef _OS_WINDOWS_ IMAGEHLP_MODULE64 ModuleInfo; ModuleInfo.SizeOfStruct = sizeof(IMAGEHLP_MODULE64); + jl_refresh_dbg_module_list(); jl_in_stackwalk = 1; bool isvalid = SymGetModuleInfo64(GetCurrentProcess(), (DWORD64)pointer, &ModuleInfo); jl_in_stackwalk = 0; diff --git a/src/stackwalk.c b/src/stackwalk.c index e67308ea01b25..f45b840a0c24c 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -194,17 +194,21 @@ static DWORD64 WINAPI JuliaGetModuleBase64( #endif } +// Might be called from unmanaged thread. int needsSymRefreshModuleList; BOOL (WINAPI *hSymRefreshModuleList)(HANDLE); -static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) +void jl_refresh_dbg_module_list(void) { - // Might be called from unmanaged thread. if (needsSymRefreshModuleList && hSymRefreshModuleList != 0 && !jl_in_stackwalk) { jl_in_stackwalk = 1; hSymRefreshModuleList(GetCurrentProcess()); jl_in_stackwalk = 0; needsSymRefreshModuleList = 0; } +} +static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) +{ + jl_refresh_dbg_module_list(); #if !defined(_CPU_X86_64_) if (jl_in_stackwalk) { return 0; @@ -372,7 +376,7 @@ JL_DLLEXPORT jl_value_t *jl_lookup_code_address(void *ip, int skipC) jl_svecset(r, 3, frame.linfo != NULL ? (jl_value_t*)frame.linfo : jl_nothing); jl_svecset(r, 4, jl_box_bool(frame.fromC)); jl_svecset(r, 5, jl_box_bool(frame.inlined)); - jl_svecset(r, 6, jl_box_long((intptr_t)ip)); + jl_svecset(r, 6, jl_box_voidpointer(ip)); } free(frames); JL_GC_POP(); diff --git a/test/stacktraces.jl b/test/stacktraces.jl index 2f36384a11112..2ba19b597b3de 100644 --- a/test/stacktraces.jl +++ b/test/stacktraces.jl @@ -109,3 +109,13 @@ let li = typeof(getfield).name.mt.cache.func::LambdaInfo, repr = string(sf) @test repr == " in getfield(...) at b:3" end + +let ctestptr = cglobal((:ctest, "libccalltest")), + ctest = StackTraces.lookup(ctestptr + 1) + + @test length(ctest) == 1 + @test ctest[1].func === :ctest + @test isnull(ctest[1].linfo) + @test ctest[1].from_c + @test ctest[1].pointer === UInt64(ctestptr) +end From 9a89ff9b81294efdc97348e768a4f5e0bed6db03 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Wed, 17 Aug 2016 13:00:19 -0700 Subject: [PATCH 054/114] Add missing tests for diagonal and uniformscaling (#17921) * Add missing tests for diagonal and uniformscaling * Change A_mul_B to * * Moved things out of nested loop that don't need to be there (cherry picked from commit ee80a0c3eb65d5ab07080b4007f2273e5dcc1259) --- test/linalg/diagonal.jl | 183 ++++++++++++++++++---------------- test/linalg/uniformscaling.jl | 2 + 2 files changed, 100 insertions(+), 85 deletions(-) diff --git a/test/linalg/diagonal.jl b/test/linalg/diagonal.jl index be856257cef9d..6d87b0f97b3cb 100644 --- a/test/linalg/diagonal.jl +++ b/test/linalg/diagonal.jl @@ -36,6 +36,13 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) @test D[1,1] == d[1] @test D[1,2] == 0 + @test issymmetric(D) + @test istriu(D) + @test istril(D) + if elty <: Real + @test ishermitian(D) + end + debug && println("Simple unary functions") for op in (-,) @test op(D)==op(DM) @@ -69,6 +76,10 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) debug && println("Linear solve") @test_approx_eq_eps D*v DM*v n*eps(relty)*(elty<:Complex ? 2:1) @test_approx_eq_eps D*U DM*U n^2*eps(relty)*(elty<:Complex ? 2:1) + + @test U.'*D ≈ U.'*full(D) + @test U'*D ≈ U'*full(D) + if relty != BigFloat @test_approx_eq_eps D\v DM\v 2n^2*eps(relty)*(elty<:Complex ? 2:1) @test_approx_eq_eps D\U DM\U 2n^3*eps(relty)*(elty<:Complex ? 2:1) @@ -95,98 +106,100 @@ for relty in (Float32, Float64, BigFloat), elty in (relty, Complex{relty}) b = view(rand(elty,n+1),collect(1:n+1)) @test_throws DimensionMismatch A_ldiv_B!(D,b) end + end + end + debug && println("Binary operations") + d = convert(Vector{elty}, randn(n)) + D2 = Diagonal(d) + DM2= diagm(d) + for op in (+, -, *) + @test full(op(D, D2)) ≈ op(DM, DM2) + end + # binary ops with plain numbers + a = rand() + @test full(a*D) ≈ a*DM + @test full(D*a) ≈ DM*a + @test full(D/a) ≈ DM/a + if relty <: BlasFloat + b = rand(elty,n,n) + b = sparse(b) + @test A_mul_B!(copy(D), copy(b)) ≈ full(D)*full(b) + @test At_mul_B!(copy(D), copy(b)) ≈ full(D).'*full(b) + @test Ac_mul_B!(copy(D), copy(b)) ≈ full(D)'*full(b) + end - debug && println("Binary operations") - d = convert(Vector{elty}, randn(n)) - D2 = Diagonal(d) - DM2= diagm(d) - for op in (+, -, *) - @test full(op(D, D2)) ≈ op(DM, DM2) - end - # binary ops with plain numbers - a = rand() - @test full(a*D) ≈ a*DM - @test full(D*a) ≈ DM*a - @test full(D/a) ≈ DM/a - if relty <: BlasFloat - b = rand(elty,n,n) - b = sparse(b) - @test A_mul_B!(copy(D), copy(b)) ≈ full(D)*full(b) - @test At_mul_B!(copy(D), copy(b)) ≈ full(D).'*full(b) - @test Ac_mul_B!(copy(D), copy(b)) ≈ full(D)'*full(b) - end + #a few missing mults + bd = Bidiagonal(D2) + @test D*D2.' ≈ full(D)*full(D2).' + @test D2*D.' ≈ full(D2)*full(D).' + @test D2*D' ≈ full(D2)*full(D)' - @test U.'*D ≈ U.'*full(D) - @test U'*D ≈ U'*full(D) + #division of two Diagonals + @test D/D2 ≈ Diagonal(D.diag./D2.diag) + @test D\D2 ≈ Diagonal(D2.diag./D.diag) + # test triu/tril + @test istriu(D) + @test istril(D) + @test triu(D,1) == zeros(D) + @test triu(D,0) == D + @test triu(D,-1) == D + @test tril(D,1) == D + @test tril(D,-1) == zeros(D) + @test tril(D,0) == D + @test_throws ArgumentError tril(D,n+1) + @test_throws ArgumentError triu(D,n+1) - #division of two Diagonals - @test D/D2 ≈ Diagonal(D.diag./D2.diag) - @test D\D2 ≈ Diagonal(D2.diag./D.diag) - # test triu/tril - @test istriu(D) - @test istril(D) - @test triu(D,1) == zeros(D) - @test triu(D,0) == D - @test triu(D,-1) == D - @test tril(D,1) == D - @test tril(D,-1) == zeros(D) - @test tril(D,0) == D - @test_throws ArgumentError tril(D,n+1) - @test_throws ArgumentError triu(D,n+1) - - # factorize - @test factorize(D) == D - - debug && println("Eigensystem") - eigD = eigfact(D) - @test Diagonal(eigD[:values]) ≈ D - @test eigD[:vectors] == eye(D) - - debug && println("ldiv") - v = rand(n + 1) - @test_throws DimensionMismatch D\v - v = rand(n) - @test D\v ≈ DM\v - V = rand(n + 1, n) - @test_throws DimensionMismatch D\V - V = rand(n, n) - @test D\V ≈ DM\V - - debug && println("conj and transpose") - @test transpose(D) == D - if elty <: BlasComplex - @test full(conj(D)) ≈ conj(DM) - @test ctranspose(D) == conj(D) - end + # factorize + @test factorize(D) == D - #logdet - if relty <: Real - ld=convert(Vector{relty},rand(n)) - @test logdet(Diagonal(ld)) ≈ logdet(diagm(ld)) - end + debug && println("Eigensystem") + eigD = eigfact(D) + @test Diagonal(eigD[:values]) ≈ D + @test eigD[:vectors] == eye(D) - #similar - @test isa(similar(D), Diagonal{elty}) - @test isa(similar(D, Int), Diagonal{Int}) - @test isa(similar(D, (3,2)), Matrix{elty}) - @test isa(similar(D, Int, (3,2)), Matrix{Int}) - - #10036 - @test issymmetric(D2) - @test ishermitian(D2) - if elty <: Complex - dc = d + im*convert(Vector{elty}, ones(n)) - D3 = Diagonal(dc) - @test issymmetric(D3) - @test !ishermitian(D3) - end + debug && println("ldiv") + v = rand(n + 1) + @test_throws DimensionMismatch D\v + v = rand(n) + @test D\v ≈ DM\v + V = rand(n + 1, n) + @test_throws DimensionMismatch D\V + V = rand(n, n) + @test D\V ≈ DM\V - U, s, V = svd(D) - @test (U*Diagonal(s))*V' ≈ D - @test svdvals(D) == s - @test svdfact(D)[:V] == V - end + debug && println("conj and transpose") + @test transpose(D) == D + if elty <: BlasComplex + @test full(conj(D)) ≈ conj(DM) + @test ctranspose(D) == conj(D) end + + #logdet + if relty <: Real + ld=convert(Vector{relty},rand(n)) + @test logdet(Diagonal(ld)) ≈ logdet(diagm(ld)) + end + + #similar + @test isa(similar(D), Diagonal{elty}) + @test isa(similar(D, Int), Diagonal{Int}) + @test isa(similar(D, (3,2)), Matrix{elty}) + @test isa(similar(D, Int, (3,2)), Matrix{Int}) + + #10036 + @test issymmetric(D2) + @test ishermitian(D2) + if elty <: Complex + dc = d + im*convert(Vector{elty}, ones(n)) + D3 = Diagonal(dc) + @test issymmetric(D3) + @test !ishermitian(D3) + end + + U, s, V = svd(D) + @test (U*Diagonal(s))*V' ≈ D + @test svdvals(D) == s + @test svdfact(D)[:V] == V end D = Diagonal(Matrix{Float64}[randn(3,3), randn(2,2)]) diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index 4d1c3e20d0ad8..f20dd0315cab8 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -8,8 +8,10 @@ srand(123) @test I[1,1] == 1 # getindex @test I[1,2] == 0 # getindex @test I === I' # transpose +@test ndims(I) == 2 @test one(UniformScaling{Float32}) == UniformScaling(one(Float32)) @test zero(UniformScaling{Float32}) == UniformScaling(zero(Float32)) +@test eltype(one(UniformScaling{Float32})) == Float32 @test zero(UniformScaling(rand(Complex128))) == zero(UniformScaling{Complex128}) @test one(UniformScaling(rand(Complex128))) == one(UniformScaling{Complex128}) @test eltype(one(UniformScaling(rand(Complex128)))) == Complex128 From 32610e288bf9f534ac3eaa4d9fa2bfff495fa048 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 16 Aug 2016 13:33:01 -0400 Subject: [PATCH 055/114] assume `getfield` is non-volatile if inferred to be constant fixes regression caused by #18017 (cherry picked from commit e35a7f65aa9bc92c56e57af91837f7810b9092be) ref #18058 --- base/inference.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index 95471df20d788..b1fbf454b7715 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2257,7 +2257,7 @@ function effect_free(e::ANY, linfo::LambdaInfo, allow_volatile::Bool) if !allow_volatile if is_known_call(e, arrayref, linfo) || is_known_call(e, arraylen, linfo) return false - elseif is_known_call(e, getfield, linfo) + elseif is_known_call(e, getfield, linfo) && !isa(exprtype(e,linfo), Const) # first argument must be immutable to ensure e is affect_free a = ea[2] typ = widenconst(exprtype(a, linfo)) From caac5818d22f010d57b6f4e8cfd5996360469807 Mon Sep 17 00:00:00 2001 From: mcprentiss Date: Wed, 17 Aug 2016 22:15:45 -0500 Subject: [PATCH 056/114] updates to README.md (#18093) This link has moved. (cherry picked from commit 0f805fb2a4cc63cd0ba7ae0d5ec1048bc7223c89) --- test/perf/shootout/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/perf/shootout/README b/test/perf/shootout/README index 948bb7eeb447c..ec8c91df1749c 100644 --- a/test/perf/shootout/README +++ b/test/perf/shootout/README @@ -1,6 +1,6 @@ This directory contains the Julia version of the "The Computer Language Benchmarks Game": -http://shootout.alioth.debian.org/ +https://benchmarksgame.alioth.debian.org/ The source code for all the benchmarks are available there: http://alioth.debian.org/scm/viewvc.php/shootout/bench/?root=shootout From c776bbc74b73c9df546b11284ca230cdb3e0e878 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 17 Aug 2016 00:06:21 -0400 Subject: [PATCH 057/114] move recheck_tuple_intersection to thread-local (cherry picked from commit 575f2aee4cf49f6f5e830284c84d135f2b9eea77) ref #18094 --- src/jltypes.c | 78 +++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index c6eedef9f8b37..b50ebdb2e94ca 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -392,10 +392,14 @@ static void extend(jl_value_t *var, jl_value_t *val, cenv_t *soln) } static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var); + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var); static jl_value_t *intersect_union(jl_uniontype_t *a, jl_value_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var) { int eq0 = eqc->n, co0 = penv->n; size_t i, l = jl_svec_len(a->types); @@ -411,11 +415,11 @@ static jl_value_t *intersect_union(jl_uniontype_t *a, jl_value_t *b, for(i=0; i < l; i++) { int eq_l = eqc->n, co_l = penv->n; jl_value_t *ti = jl_type_intersect(jl_svecref(a->types,i), b, - penv, eqc, var); + penv, eqc, recheck_tuple_intersection, var); if (ti == (jl_value_t*)jl_bottom_type) { eqc->n = eq0; penv->n = co0; ti = jl_type_intersect(jl_svecref(a->types,i), b, - penv, eqc, var); + penv, eqc, recheck_tuple_intersection, var); if (ti != (jl_value_t*)jl_bottom_type) { // tvar conflict among union elements; keep the conflicting // constraints rolled back @@ -553,10 +557,10 @@ to be returned, with one exception illustrated by: where typeintersect(B,C) == Bottom. */ -int recheck_tuple_intersection = 0; // "flag" above - static jl_value_t *intersect_tuple(jl_datatype_t *a, jl_datatype_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, // "flag" above + variance_t var) { jl_svec_t *ap = a->parameters, *bp = b->parameters; size_t alenr = jl_svec_len(ap), blenr = jl_svec_len(bp); @@ -590,7 +594,7 @@ static jl_value_t *intersect_tuple(jl_datatype_t *a, jl_datatype_t *b, // Do we need to store "at least N" constraints in penv? // Formerly, typeintersect(Tuple{A,Vararg{B}}, NTuple{N,C}) did that if (akind == JL_VARARG_BOUND || bkind == JL_VARARG_BOUND) - recheck_tuple_intersection = 1; + *recheck_tuple_intersection = 1; } if (bottom) return (jl_value_t*) jl_bottom_type; if (n == 0) return jl_typeof(jl_emptytuple); @@ -626,17 +630,17 @@ static jl_value_t *intersect_tuple(jl_datatype_t *a, jl_datatype_t *b, bi++; } assert(ae!=NULL && be!=NULL); - ce = jl_type_intersect(ae,be,penv,eqc,var); + ce = jl_type_intersect(ae, be, penv, eqc, recheck_tuple_intersection, var); if (ce == (jl_value_t*)jl_bottom_type) { if (var!=invariant && aseq && bseq) { // (X∩Y)==∅ → (X...)∩(Y...) == () // We don't need to set bindings here because - // recheck_tuple_intersection=1 + // *recheck_tuple_intersection = 1 if (n == 1) { JL_GC_POP(); return (jl_value_t*)jl_typeof(jl_emptytuple); } - jl_svec_set_len_unsafe(tc,jl_svec_len(tc)-1); + jl_svec_set_len_unsafe(tc, jl_svec_len(tc) - 1); goto done_intersect_tuple; } JL_GC_POP(); @@ -653,7 +657,9 @@ static jl_value_t *intersect_tuple(jl_datatype_t *a, jl_datatype_t *b, } static jl_value_t *intersect_tag(jl_datatype_t *a, jl_datatype_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var) { assert(a->name == b->name); assert(jl_svec_len(a->parameters) == jl_svec_len(b->parameters)); @@ -678,7 +684,7 @@ static jl_value_t *intersect_tag(jl_datatype_t *a, jl_datatype_t *b, return (jl_value_t*)jl_bottom_type; } } - ti = jl_type_intersect(ap,bp,penv,eqc,invariant); + ti = jl_type_intersect(ap, bp, penv, eqc, recheck_tuple_intersection, invariant); if (bp == (jl_value_t*)jl_bottom_type && !((jl_tvar_t*)ap)->bound) { // "Union{}" as a type parameter @@ -687,7 +693,7 @@ static jl_value_t *intersect_tag(jl_datatype_t *a, jl_datatype_t *b, } } else if (jl_is_typevar(bp)) { - ti = jl_type_intersect(ap,bp,penv,eqc,invariant); + ti = jl_type_intersect(ap, bp, penv, eqc, recheck_tuple_intersection, invariant); if (ap == (jl_value_t*)jl_bottom_type && !((jl_tvar_t*)bp)->bound) { // "Union{}" as a type parameter @@ -701,7 +707,7 @@ static jl_value_t *intersect_tag(jl_datatype_t *a, jl_datatype_t *b, if (tva || tvb) { if (jl_subtype_invariant(ap,bp,0) || jl_subtype_invariant(bp,ap,0)) { - ti = jl_type_intersect(ap,bp,penv,eqc,invariant); + ti = jl_type_intersect(ap, bp, penv, eqc, recheck_tuple_intersection, invariant); } else { ti = (jl_value_t*)jl_bottom_type; @@ -786,7 +792,9 @@ static int match_intersection_mode = 0; static jl_value_t *meet_tvars(jl_tvar_t *a, jl_tvar_t *b); static jl_value_t *intersect_typevar(jl_tvar_t *a, jl_value_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var) { jl_value_t *both=NULL; jl_tvar_t *new_b=NULL; @@ -828,7 +836,7 @@ static jl_value_t *intersect_typevar(jl_tvar_t *a, jl_value_t *b, } } else { - b = jl_type_intersect(a->ub, b, penv, eqc, covariant); + b = jl_type_intersect(a->ub, b, penv, eqc, recheck_tuple_intersection, covariant); if (b == jl_bottom_type) { JL_GC_POP(); return b; @@ -924,7 +932,7 @@ static jl_value_t *intersect_typevar(jl_tvar_t *a, jl_value_t *b, return (jl_value_t*)a; } -static jl_value_t *approxify_type(jl_datatype_t *dt, jl_svec_t *pp) +static jl_value_t *approxify_type(jl_datatype_t *dt, jl_svec_t *pp, int *recheck_tuple_intersection) { size_t i, l = jl_svec_len(dt->parameters); jl_svec_t *p = jl_alloc_svec(l); @@ -942,7 +950,9 @@ static jl_value_t *approxify_type(jl_datatype_t *dt, jl_svec_t *pp) } static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, - cenv_t *penv, cenv_t *eqc, variance_t var) + cenv_t *penv, cenv_t *eqc, + int *recheck_tuple_intersection, + variance_t var) { if (jl_is_typector(a)) a = (jl_value_t*)((jl_typector_t*)a)->body; @@ -953,13 +963,13 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, if (var == covariant && !((jl_tvar_t*)a)->bound) a = ((jl_tvar_t*)a)->ub; else if (a != jl_ANY_flag) - return intersect_typevar((jl_tvar_t*)a, b, penv, eqc, var); + return intersect_typevar((jl_tvar_t*)a, b, penv, eqc, recheck_tuple_intersection, var); } if (jl_is_typevar(b)) { if (var == covariant && !((jl_tvar_t*)b)->bound) b = ((jl_tvar_t*)b)->ub; else if (b != jl_ANY_flag) - return intersect_typevar((jl_tvar_t*)b, a, penv, eqc, var); + return intersect_typevar((jl_tvar_t*)b, a, penv, eqc, recheck_tuple_intersection, var); } if (a == (jl_value_t*)jl_bottom_type || b == (jl_value_t*)jl_bottom_type) return (jl_value_t*)jl_bottom_type; @@ -971,19 +981,19 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, } // union if (jl_is_uniontype(a)) - return intersect_union((jl_uniontype_t*)a, b, penv, eqc, var); + return intersect_union((jl_uniontype_t*)a, b, penv, eqc, recheck_tuple_intersection, var); if (jl_is_uniontype(b)) - return intersect_union((jl_uniontype_t*)b, a, penv, eqc, var); + return intersect_union((jl_uniontype_t*)b, a, penv, eqc, recheck_tuple_intersection, var); if (a == (jl_value_t*)jl_any_type || a == jl_ANY_flag) return b; if (b == (jl_value_t*)jl_any_type || b == jl_ANY_flag) return a; // tuple if (jl_is_tuple_type(a)) { if (jl_is_tuple_type(b)) { - return intersect_tuple((jl_datatype_t*)a, (jl_datatype_t*)b, penv,eqc,var); + return intersect_tuple((jl_datatype_t*)a, (jl_datatype_t*)b, penv, eqc, recheck_tuple_intersection, var); } } if (jl_is_tuple_type(b)) { - return jl_type_intersect(b, a, penv,eqc,var); + return jl_type_intersect(b, a, penv, eqc, recheck_tuple_intersection, var); } // tag if (!jl_is_datatype(a) || !jl_is_datatype(b)) @@ -991,7 +1001,7 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, jl_datatype_t *tta = (jl_datatype_t*)a; jl_datatype_t *ttb = (jl_datatype_t*)b; if (tta->name == ttb->name) - return (jl_value_t*)intersect_tag(tta, ttb, penv, eqc, var); + return (jl_value_t*)intersect_tag(tta, ttb, penv, eqc, recheck_tuple_intersection, var); jl_datatype_t *super = NULL; jl_datatype_t *sub = NULL; jl_value_t *env = NULL; @@ -1055,10 +1065,10 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, if (var == covariant && sub == (jl_datatype_t*)sub->name->primary && jl_has_typevars_from((jl_value_t*)sub->super, ((jl_datatype_t*)sub->name->primary)->parameters)) - env = approxify_type((jl_datatype_t*)sub->super, ((jl_datatype_t*)sub->name->primary)->parameters); + env = approxify_type((jl_datatype_t*)sub->super, ((jl_datatype_t*)sub->name->primary)->parameters, recheck_tuple_intersection); else env = (jl_value_t*)sub->super; - super = (jl_datatype_t*)jl_type_intersect((jl_value_t*)env, (jl_value_t*)super, penv, eqc, var); + super = (jl_datatype_t*)jl_type_intersect((jl_value_t*)env, (jl_value_t*)super, penv, eqc, recheck_tuple_intersection, var); if ((jl_value_t*)super == jl_bottom_type) { JL_GC_POP(); @@ -1128,7 +1138,7 @@ static jl_value_t *jl_type_intersect(jl_value_t *a, jl_value_t *b, for(int e=0; e < jl_svec_len(env); e+=2) { if (jl_svecref(env, e) == tp) { elt = jl_type_intersect(elt, jl_svecref(env, e+1), - penv, eqc, invariant); + penv, eqc, recheck_tuple_intersection, invariant); // note: elt might be Union{} if "Union{}" was the type parameter break; } @@ -1490,14 +1500,14 @@ jl_value_t *jl_type_intersection_matching(jl_value_t *a, jl_value_t *b, jl_value_t **pti = &rts[0]; jl_value_t **extraroot = &rts[1]; - recheck_tuple_intersection = 0; + int recheck_tuple_intersection = 0; JL_TRY { // This is kind of awful, but an inner call to instantiate_type // might fail due to a mismatched type parameter. The problem is // that we allow Range{T} to exist, even though the declaration of // Range specifies Range{T<:Real}. Therefore intersection cannot see // that some parameter values actually don't match. - *pti = jl_type_intersect(a, b, &env, &eqc, covariant); + *pti = jl_type_intersect(a, b, &env, &eqc, &recheck_tuple_intersection, covariant); } JL_CATCH { *pti = (jl_value_t*)jl_bottom_type; @@ -1511,8 +1521,8 @@ jl_value_t *jl_type_intersection_matching(jl_value_t *a, jl_value_t *b, int e; if (recheck_tuple_intersection) { - for(e=0; e < eqc.n; e+=2) { - jl_value_t *val = eqc.data[e+1]; + for (e = 0; e < eqc.n; e += 2) { + jl_value_t *val = eqc.data[e + 1]; if (jl_is_long(val)) break; } @@ -1526,7 +1536,7 @@ jl_value_t *jl_type_intersection_matching(jl_value_t *a, jl_value_t *b, to find all other constraints on N first, then do intersection again with that knowledge. */ - *pti = jl_type_intersect(a, b, &env, &eqc, covariant); + *pti = jl_type_intersect(a, b, &env, &eqc, &recheck_tuple_intersection, covariant); if (*pti == (jl_value_t*)jl_bottom_type) { JL_GC_POP(); return *pti; From 2cb13f31ed36bd062e7ef246a1bb13dfe4c50ac5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 17 Aug 2016 00:42:44 -0400 Subject: [PATCH 058/114] ensure in_typeinf_loop is only read by the thread with the typeinf lock (cherry picked from commit b6b26df84aff0808735871310e84e602fee8d681) ref #18094 --- base/inference.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base/inference.jl b/base/inference.jl index b1fbf454b7715..e8b319573322c 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1469,7 +1469,11 @@ function typeinf_edge(method::Method, atypes::ANY, sparams::SimpleVector, needtr end end - if caller === nothing && in_typeinf_loop + ccall(:jl_typeinf_begin, Void, ()) + thread_in_typeinf_loop = in_typeinf_loop::Bool + ccall(:jl_typeinf_end, Void, ()) + + if caller === nothing && thread_in_typeinf_loop # if the caller needed the ast, but we are already in the typeinf loop # then just return early -- we can't fulfill this request # if the client was inlining, then this means we decided not to try to infer this From 190064fd9b446a8c6a34c424bf7b282e1d32e0d8 Mon Sep 17 00:00:00 2001 From: Katie Hyatt Date: Wed, 17 Aug 2016 13:38:07 -0700 Subject: [PATCH 059/114] Add tests for flipdim, unary ops (cherry picked from commit 1df4454b5dc5012332bc510b7ccb5bd6ada60178) ref #18092 --- test/abstractarray.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index b3ff1b2f18c77..b0f3ae07526ae 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -709,3 +709,16 @@ let @test !issparse(m1) @test !issparse(m2) end + +#isinteger and isreal +@test isinteger(Diagonal(rand(1:5,5))) +@test isreal(Diagonal(rand(5))) + +#unary ops +let A = Diagonal(rand(1:5,5)) + @test +(A) == A + @test *(A) == A +end + +#flipdim on empty +@test flipdim(Diagonal([]),1) == Diagonal([]) From beeb6c051709c1c0291a219cead3dd474c2b24a1 Mon Sep 17 00:00:00 2001 From: Katie Hyatt Date: Wed, 17 Aug 2016 13:38:17 -0700 Subject: [PATCH 060/114] Add tests for iteration methods on numbers (cherry picked from commit 07ee0457ef1ee41971e7c3dc36ec5837c1cba112) ref #18092 --- test/numbers.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/numbers.jl b/test/numbers.jl index a0ccc138e776f..ae834f109a376 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2772,11 +2772,17 @@ testmi(map(UInt32, 0:1000), map(UInt32, 1:100)) testmi(typemax(UInt32)-UInt32(1000):typemax(UInt32), map(UInt32, 1:100)) @test ndims(1) == 0 +@test ndims(Integer) == 0 @test size(1,1) == 1 @test_throws BoundsError size(1,-1) @test indices(1) == () @test indices(1,1) == 1:1 @test_throws BoundsError indices(1,-1) +@test isinteger(Integer(2)) == true +@test size(1) == () +@test length(1) == 1 +@test endof(1) == 1 +@test eltype(Integer) == Integer # issue #15920 @test Rational(0, 1) / Complex(3, 2) == 0 From 25be0c160dc0a12e70b08a8224c348282ea61568 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Wed, 17 Aug 2016 22:21:49 -0700 Subject: [PATCH 061/114] Added test for complex on SharedArrays (#18097) (cherry picked from commit 68ba177ee70f5bf39d0ca22f2164bb20c4657177) --- test/parallel_exec.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/parallel_exec.jl b/test/parallel_exec.jl index a5be3fa85915d..faeb7d165ae14 100644 --- a/test/parallel_exec.jl +++ b/test/parallel_exec.jl @@ -473,6 +473,18 @@ for (x,i) in enumerate(d) @test x == i end +# complex +sd = SharedArray(Int,10) +se = SharedArray(Int,10) +@sync @parallel for i=1:10 + sd[i] = i + se[i] = i +end +sc = complex(sd,se) +for (x,i) in enumerate(sc) + @test i == complex(x,x) +end + # Once finalized accessing remote references and shared arrays should result in exceptions. function finalize_and_test(r) finalize(r) From dd604d2cbad3ebe45786d3f0fd0eea659429bb34 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Wed, 17 Aug 2016 23:43:57 -0700 Subject: [PATCH 062/114] Set LIB_INSTALL_DIR for cmake-based projects (#18047) * Set LIB_INSTALL_DIR for cmake-based projects This fixes `MULTIARCH_INSTALL=1` installations that place libraries not into `/lib` but into `/lib/`, like Debian and Ubuntu. Without this change, `libgit2`, `libssh2` and `mbedtls` would place libraries into `/lib`, silently failing until the Julia bootstrap process attempted to dlopen one of them. * Apply `LIB_INSTALL_DIR` to `CMAKE_COMMON` (cherry picked from commit 1af5356a301f671ab57ff5f534a00c7eca96435a) --- deps/Makefile | 2 +- deps/libssh2.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 37b4defd90884..1b799d922aaee 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -46,7 +46,7 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXX CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS) CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS) -CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) +CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) -DLIB_INSTALL_DIR=$(build_shlibdir) ifneq ($(VERBOSE), 0) CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON endif diff --git a/deps/libssh2.mk b/deps/libssh2.mk index be13f58c8653e..eb2282a882e12 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -8,7 +8,7 @@ LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT) LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT) LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_BUILD_TYPE=Release ifeq ($(OS),WINNT) LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF From e313fc9d3b0a63308f97dff05b77fdb63db97f15 Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Thu, 18 Aug 2016 06:39:14 -0700 Subject: [PATCH 063/114] Missing tests for strings (#18089) (cherry picked from commit 3feef68dc6d5858b8f9ff3c97445cdf7503da3a7) --- test/strings/basic.jl | 5 +++++ test/strings/search.jl | 2 ++ 2 files changed, 7 insertions(+) diff --git a/test/strings/basic.jl b/test/strings/basic.jl index 1008ce9566a17..e736da9e69890 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -7,6 +7,11 @@ let d = [0x61,0x62,0x63,0x21] end @test String("abc!") == "abc!" +@test isempty(string()) +@test eltype(GenericString) == Char +@test start("abc") == 1 +@test cmp("ab","abc") == -1 + # {starts,ends}with @test startswith("abcd", 'a') @test startswith("abcd", "a") diff --git a/test/strings/search.jl b/test/strings/search.jl index 6cc7b7233619a..8fcfe9c62d5d3 100644 --- a/test/strings/search.jl +++ b/test/strings/search.jl @@ -373,3 +373,5 @@ end # string searchindex with a two-char UTF-8 (4 byte) string literal @test rsearchindex("\U1f596\U1f596", "\U1f596\U1f596") == 1 @test rsearchindex("\U1f596\U1f596", "\U1f596\U1f596", endof("\U1f596\U1f596\U1f596")) == 1 + +@test_throws ErrorException "ab" ∈ "abc" From cb1e04d191555c88dc13986d6f51898e942b4337 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 18 Aug 2016 20:08:17 +0800 Subject: [PATCH 064/114] Fix compiler warnings on 32bits (cherry picked from commit e7110971dd7e8076ee15aacc28e3e595244b36cc) ref #18108 --- src/llvm-ptls.cpp | 1 + src/support/hashing.h | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 019975cc3ba05..a3d669a394655 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -147,6 +147,7 @@ void LowerPTLS::runOnFunction(LLVMContext &ctx, Module &M, Function *F, Value *tls = nullptr; assert(0 && "Cannot emit thread pointer for this architecture."); # endif + (void)T_pint8; ptlsStates->replaceAllUsesWith(tls); ptlsStates->eraseFromParent(); } diff --git a/src/support/hashing.h b/src/support/hashing.h index 1be2a9e7b5ec0..5f9b41d6f700d 100644 --- a/src/support/hashing.h +++ b/src/support/hashing.h @@ -3,6 +3,9 @@ #ifndef HASHING_H #define HASHING_H +#include "utils.h" +#include "dtypes.h" + #ifdef __cplusplus extern "C" { #endif @@ -22,10 +25,17 @@ JL_DLLEXPORT uint32_t memhash32(const char *buf, size_t n); JL_DLLEXPORT uint32_t memhash32_seed(const char *buf, size_t n, uint32_t seed); #ifdef _P64 -#define bitmix(a,b) int64hash((a)^bswap_64(b)) +STATIC_INLINE uint64_t bitmix(uint64_t a, uint64_t b) +{ + return int64hash(a^bswap_64(b)); +} #else -#define bitmix(a,b) int64to32hash((((uint64_t)a)<<32)|((uint64_t)b)) +STATIC_INLINE uint32_t bitmix(uint32_t a, uint32_t b) +{ + return int64to32hash((((uint64_t)a) << 32) | (uint64_t)b); +} #endif +#define bitmix(a, b) (bitmix)((uintptr_t)(a), (uintptr_t)(b)) #ifdef __cplusplus } From ddddd4dac3fc437ca3473d4758ec1bb262c6b405 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 16 Aug 2016 19:48:03 +0000 Subject: [PATCH 065/114] improve FreeBSD / generic posix support (cherry picked from commit 6a62d300313a80ff0d435481e861e06ff32a5890) ref #18063 Conflicts: src/signals-unix.c Not including changes to jl_call_in_ctx in this file because those changes were part of https://github.com/JuliaLang/julia/pull/17727 which has not been backported to release-0.5 at this time. --- Make.inc | 5 +++++ README.md | 16 +++++++++++++--- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + deps/libgit2.mk | 7 ++++++- deps/libuv.version | 2 +- src/cgmemmgr.cpp | 2 +- src/flisp/Makefile | 4 ++-- src/gc-pages.c | 4 ++++ src/gc.c | 8 ++++++-- src/julia_threads.h | 2 +- src/safepoint.c | 2 +- src/signals-unix.c | 2 +- src/task.c | 2 +- 16 files changed, 44 insertions(+), 16 deletions(-) delete mode 100644 deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 delete mode 100644 deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 create mode 100644 deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/md5 create mode 100644 deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/sha512 diff --git a/Make.inc b/Make.inc index 003f50db78376..b8b04c9332313 100644 --- a/Make.inc +++ b/Make.inc @@ -340,6 +340,11 @@ endif STDLIBCPP_FLAG := +ifeq ($(OS), FreeBSD) +USEGCC := 0 +USECLANG := 1 +endif + ifeq ($(OS), Darwin) DARWINVER := $(shell uname -r | cut -b 1-2) DARWINVER_GTE13 := $(shell expr `uname -r | cut -b 1-2` \>= 13) diff --git a/README.md b/README.md index d893f1b32c036..aee51ca376a81 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ developers may find the notes in [CONTRIBUTING](https://github.com/JuliaLang/jul - **FreeBSD** - **Windows** -All systems are supported with both x86/64 (64-bit) and x86 (32-bit) architectures. Experimental and early support for [ARM](https://github.com/JuliaLang/julia/blob/master/README.arm.md) is available too. +All systems are supported with both x86/64 (64-bit) and x86 (32-bit) architectures. Experimental and early support for [ARM](https://github.com/JuliaLang/julia/blob/master/README.arm.md), AARCH64, and POWER (little-endian) is available too. ## Source Download and Compilation @@ -235,12 +235,22 @@ When building Julia, or its dependencies, libraries installed by third party pac ### FreeBSD -On *FreeBSD Release 9.0*, install the `gcc47`, `git`, and `gmake` packages/ports, and compile Julia with the command: +On *FreeBSD Release 11.0*, install the gfortran, git, cmake, and gmake packages/ports (`pkg install gcc6 gmake git cmake`), and compile Julia with the command: - $ gmake FC=gfortran47 + $ echo 'FC=gfortran6' >> Make.user + $ gmake You must use the `gmake` command on FreeBSD instead of `make`. +Note that Julia is community-supported and we have little control over our upstream dependencies, you may still run into issues with its dependencies and YMMV. Current known issues includes: + + - The x86 arch doesn't support threading due to lack of compiler runtime library support (set JULIA_THREADS=0). + - libunwind needs a small patch to its tests to compile. + - OpenBLAS patches in pkg haven't been upstreamed. + - gfortran can't link binaries. Set `FFLAGS=-Wl,-rpath,/usr/local/lib/gcc6` to workaround this (upstream bug submitted to FreeBSD pkg maintainers). + - System libraries installed by pkg are not on the compiler path by default. You may need to add `LDFLAGS=/usr/local/lib` and `CPPFLAGS=/usr/local/include` to your environment or Make.user file to build successfully. + + ### Windows In order to build Julia on Windows, see [README.windows](https://github.com/JuliaLang/julia/blob/master/README.windows.md). diff --git a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 deleted file mode 100644 index e42a20ffd5a35..0000000000000 --- a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -cc07a8ef026fa42eeaddf7b6e074096e diff --git a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 b/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 deleted file mode 100644 index bec2d8d44203f..0000000000000 --- a/deps/checksums/libuv-28743d6091531340cfe316de2b2d385fe1778ff5.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -e128cec9548ff2f52a78743203d3f06dceef3cf7cbeb3a7a5f7453b132576833f82688eed52cf74c035f57828deac079cd845ad47c8cd6197a8e0bebf3586fc2 diff --git a/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/md5 b/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/md5 new file mode 100644 index 0000000000000..89795f7987bbc --- /dev/null +++ b/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/md5 @@ -0,0 +1 @@ +4c91d4c9161555c911630b0a70ddec03 diff --git a/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/sha512 b/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/sha512 new file mode 100644 index 0000000000000..faf62292a62af --- /dev/null +++ b/deps/checksums/libuv-8d5131b6c1595920dd30644cd1435b4f344b46c8.tar.gz/sha512 @@ -0,0 +1 @@ +c53513a5aea84405bf302b084a23f24f54148aac90a2bd666219ce14879723baab959942934d0d801a4572fffd07e60a7d574ade8d7eb57b6da8216063c20a48 diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 0e530132f7732..f17bea8ff3345 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -47,9 +47,14 @@ ifeq ($(OS),Linux) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-require-openssl.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-openssl-hang.patch-applied endif ifneq ($(OS),WINNT) +ifeq ($(USE_SYSTEM_CURL), 0) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(CURL_OBJ_TARGET) endif -$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied $(LIBSSH2_OBJ_TARGET) +endif +ifeq ($(USE_SYSTEM_LIBSSH2), 0) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(LIBSSH2_OBJ_TARGET) +endif +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-ssh.patch-applied $(SRCDIR)/srccache/$(LIBGIT2_SRC_DIR)/libgit2-agent-nonfatal.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) diff --git a/deps/libuv.version b/deps/libuv.version index 478ebca66589a..4ba4b36712aaf 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv1.9.0 -LIBUV_SHA1=28743d6091531340cfe316de2b2d385fe1778ff5 +LIBUV_SHA1=8d5131b6c1595920dd30644cd1435b4f344b46c8 diff --git a/src/cgmemmgr.cpp b/src/cgmemmgr.cpp index 5c3b187ecab25..a85dfa57a227b 100644 --- a/src/cgmemmgr.cpp +++ b/src/cgmemmgr.cpp @@ -47,7 +47,7 @@ static void *map_anon_page(size_t size) mem = (char*)LLT_ALIGN(uintptr_t(mem), jl_page_size); #else // _OS_WINDOWS_ void *mem = mmap(nullptr, size, PROT_READ | PROT_WRITE, - MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); assert(mem != MAP_FAILED && "Cannot allocate RW memory"); #endif // _OS_WINDOWS_ return mem; diff --git a/src/flisp/Makefile b/src/flisp/Makefile index 5d7cf6fc51ccd..5de2aff8a4888 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -77,10 +77,10 @@ CCLD := $(LD) endif $(BUILDDIR)/$(EXENAME)-debug: $(DOBJS) $(LIBFILES_debug) $(BUILDDIR)/$(LIBTARGET)-debug.a $(BUILDDIR)/flmain.dbg.obj | $(BUILDDIR)/flisp.boot - @$(call PRINT_LINK, $(CCLD) $(DEBUGFLAGS) $(DOBJS) $(BUILDDIR)/flmain.dbg.obj -o $@ $(BUILDDIR)/$(LIBTARGET)-debug.a $(LIBFILES_debug) $(LIBS) $(OSLIBS)) + @$(call PRINT_LINK, $(CCLD) $(DEBUGFLAGS) $(LDFLAGS) $(DOBJS) $(BUILDDIR)/flmain.dbg.obj -o $@ $(BUILDDIR)/$(LIBTARGET)-debug.a $(LIBFILES_debug) $(LIBS) $(OSLIBS)) $(BUILDDIR)/$(EXENAME): $(OBJS) $(LIBFILES_release) $(BUILDDIR)/$(LIBTARGET).a $(BUILDDIR)/flmain.o | $(BUILDDIR)/flisp.boot - @$(call PRINT_LINK, $(CCLD) $(SHIPFLAGS) $(OBJS) $(BUILDDIR)/flmain.o -o $@ $(BUILDDIR)/$(LIBTARGET).a $(LIBFILES_release) $(LIBS) $(OSLIBS)) + @$(call PRINT_LINK, $(CCLD) $(SHIPFLAGS) $(LDFLAGS) $(OBJS) $(BUILDDIR)/flmain.o -o $@ $(BUILDDIR)/$(LIBTARGET).a $(LIBFILES_release) $(LIBS) $(OSLIBS)) ifneq ($(BUILDDIR),.) $(BUILDDIR)/flisp.boot: flisp.boot diff --git a/src/gc-pages.c b/src/gc-pages.c index ead16010cee15..fbe2b27450fe3 100644 --- a/src/gc-pages.c +++ b/src/gc-pages.c @@ -37,6 +37,10 @@ void jl_gc_init_page(void) #endif } +#ifndef MAP_NORESERVE // not defined in POSIX, FreeBSD, etc. +#define MAP_NORESERVE (0) +#endif + // Try to allocate a memory block for a region with `pg_cnt` pages. // Return `NULL` if allocation failed. Result is aligned to `GC_PAGE_SZ`. static char *jl_gc_try_alloc_region(int pg_cnt) diff --git a/src/gc.c b/src/gc.c index b5ac77e6a7c4d..5ad35d739d318 100644 --- a/src/gc.c +++ b/src/gc.c @@ -618,7 +618,11 @@ JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) bigval_t *v = (bigval_t*)malloc_cache_align(allocsz); if (v == NULL) jl_throw(jl_memory_exception); +#ifdef JULIA_ENABLE_THREADING jl_atomic_fetch_add(&gc_num.allocd, allocsz); +#else + gc_num.allocd += allocsz; +#endif gc_num.bigalloc++; #ifdef MEMDEBUG memset(v, 0xee, allocsz); @@ -1596,7 +1600,7 @@ static void post_mark(arraylist_t *list) } // collector entry point and control -static volatile uint64_t jl_gc_disable_counter = 0; +static volatile uint32_t jl_gc_disable_counter = 0; JL_DLLEXPORT int jl_gc_enable(int on) { @@ -2087,7 +2091,7 @@ void *jl_gc_perm_alloc_nolock(size_t sz) pool = (void*)LLT_ALIGN((uintptr_t)pool, JL_SMALL_BYTE_ALIGNMENT); #else void *pool = mmap(0, GC_PERM_POOL_SIZE, PROT_READ | PROT_WRITE, - MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (__unlikely(pool == MAP_FAILED)) return NULL; #endif diff --git a/src/julia_threads.h b/src/julia_threads.h index e18eb7283e1d4..5302fa88c31d6 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -159,7 +159,7 @@ static inline unsigned long JL_CONST_FUNC jl_thread_self(void) * 2. (most importantly) we need interoperability between code written * in different languages. * The current c++ standard (c++14) does not allow using c11 atomic - * functions or types and there's currently no grantee that the two + * functions or types and there's currently no guarantee that the two * types are compatible (although most of them probably are). * We also need to access these atomic variables from the LLVM JIT code * which is very hard unless the layout of the object is fully diff --git a/src/safepoint.c b/src/safepoint.c index 6644f32b63c61..2874266ebe4b5 100644 --- a/src/safepoint.c +++ b/src/safepoint.c @@ -93,7 +93,7 @@ void jl_safepoint_init(void) char *addr = (char*)VirtualAlloc(NULL, pgsz * 3, MEM_COMMIT, PAGE_READONLY); #else char *addr = (char*)mmap(0, pgsz * 3, PROT_READ, - MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (addr == MAP_FAILED) addr = NULL; #endif diff --git a/src/signals-unix.c b/src/signals-unix.c index 9557dd0a3bd60..d0814741fe9c5 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -315,7 +315,7 @@ static void *alloc_sigstack(size_t size) // Add one guard page to catch stack overflow in the signal handler size = LLT_ALIGN(size, pagesz) + pagesz; void *stackbuff = mmap(0, size, PROT_READ | PROT_WRITE, - MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (stackbuff == MAP_FAILED) jl_errorf("fatal error allocating signal stack: mmap: %s", strerror(errno)); diff --git a/src/task.c b/src/task.c index b1cf229bd1248..2c1c249f9670e 100644 --- a/src/task.c +++ b/src/task.c @@ -353,7 +353,7 @@ static void ctx_switch(jl_ptls_t ptls, jl_task_t *t, jl_jmp_buf *where) " push %%ebp;\n" // instead of ESP " jmp %P1;\n" // call `start_task` with fake stack frame " ud2" - : : "r" (stackbase), ""(&start_task) : "memory" ); + : : "r" (stackbase), "X"(&start_task) : "memory" ); #elif defined(_CPU_AARCH64_) asm(" mov sp, %0;\n" " mov x29, xzr;\n" // Clear link register (x29) and frame pointer From 24c98c2c1351046856ac1cbe7fe4a1fbb3d0cf44 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 18 Aug 2016 04:16:26 -0700 Subject: [PATCH 066/114] readme formatting [ci skip] (cherry picked from commit d3951aa192e72f4086ca5e2e87a035940228272d) ref #18063 --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index aee51ca376a81..bf89ab33271bf 100644 --- a/README.md +++ b/README.md @@ -242,13 +242,13 @@ On *FreeBSD Release 11.0*, install the gfortran, git, cmake, and gmake packages/ You must use the `gmake` command on FreeBSD instead of `make`. -Note that Julia is community-supported and we have little control over our upstream dependencies, you may still run into issues with its dependencies and YMMV. Current known issues includes: +Note that Julia is community-supported and we have little control over our upstream dependencies, you may still run into issues with dependencies and YMMV. Current known issues include: - - The x86 arch doesn't support threading due to lack of compiler runtime library support (set JULIA_THREADS=0). + - The x86 arch doesn't support threading due to lack of compiler runtime library support (set `JULIA_THREADS=0`). - libunwind needs a small patch to its tests to compile. - OpenBLAS patches in pkg haven't been upstreamed. - - gfortran can't link binaries. Set `FFLAGS=-Wl,-rpath,/usr/local/lib/gcc6` to workaround this (upstream bug submitted to FreeBSD pkg maintainers). - - System libraries installed by pkg are not on the compiler path by default. You may need to add `LDFLAGS=/usr/local/lib` and `CPPFLAGS=/usr/local/include` to your environment or Make.user file to build successfully. + - gfortran can't link binaries. Set `FFLAGS=-Wl,-rpath,/usr/local/lib/gcc6` to work around this (upstream bug submitted to FreeBSD pkg maintainers). + - System libraries installed by pkg are not on the compiler path by default. You may need to add `LDFLAGS=/usr/local/lib` and `CPPFLAGS=/usr/local/include` to your environment or `Make.user` file to build successfully. ### Windows From edcd7f905e4c2876ca5a608e92e2af73f0cd1183 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 16 Aug 2016 11:23:27 -0500 Subject: [PATCH 067/114] Fix bug with operator precendence (cherry picked from commit 2192d192a7b5fb180821e107dff755d8053222a9) ref #18066 --- base/libgit2/callbacks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index fd2580da27ee7..cbab587516e82 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -160,7 +160,7 @@ function authenticate_userpass(creds::UserPasswordCredentials, libgit2credptr::P urlusername : username) userpass = prompt("Password for '$schema$username@$host'", password=true) end - (creds.user != username) || (creds.pass != userpass) && reset!(creds) + ((creds.user != username) || (creds.pass != userpass)) && reset!(creds) creds.user = username # save credentials creds.pass = userpass # save credentials From 95acc0f0d492430f6e69493c8d043dc2438311e7 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 16 Aug 2016 14:34:46 -0500 Subject: [PATCH 068/114] Improve URL regex (cherry picked from commit d2bbdd2cfcee768864c4b8149230a3f9b537846a) ref #18066 --- base/libgit2/callbacks.jl | 10 ++++------ base/libgit2/utils.jl | 8 +++++++- test/libgit2.jl | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/base/libgit2/callbacks.jl b/base/libgit2/callbacks.jl index cbab587516e82..13f2f88e168de 100644 --- a/base/libgit2/callbacks.jl +++ b/base/libgit2/callbacks.jl @@ -209,12 +209,10 @@ function credentials_callback(libgit2credptr::Ptr{Ptr{Void}}, url_ptr::Cstring, url = unsafe_string(url_ptr) # parse url for schema and host - urlparts = match(urlmatcher, url) - schema = urlparts.captures[1] - urlusername = urlparts.captures[4] - urlusername = urlusername === nothing ? "" : String(urlusername) - host = urlparts.captures[5] - schema = schema === nothing ? "" : schema*"://" + urlparts = match(URL_REGEX, url) + schema = urlparts[:scheme] === nothing ? "" : urlparts[:scheme] * "://" + urlusername = urlparts[:user] === nothing ? "" : urlparts[:user] + host = urlparts[:host] # get credentials object from payload pointer @assert payload_ptr != C_NULL diff --git a/base/libgit2/utils.jl b/base/libgit2/utils.jl index ba2149dbda576..ab8928687e3c7 100644 --- a/base/libgit2/utils.jl +++ b/base/libgit2/utils.jl @@ -1,6 +1,12 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license -const urlmatcher = r"^(http[s]?|git|ssh)?(:\/\/)?((\w+)@)?([A-Za-z0-9\-\.]+)(:[0-9]+)?(.*)$" +const URL_REGEX = r""" +^(?:(?https?|git|ssh)\:\/\/)? +(?:(?.*?)(?:\:(?.*?))?@)? +(?[A-Za-z0-9\-\.]+) +(?:\:(?\d+)?)? +(?.*?)$ +"""x function version() major = Ref{Cint}(0) diff --git a/test/libgit2.jl b/test/libgit2.jl index db7d4924688c9..64c7fa7f0e47a 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -76,6 +76,39 @@ const LIBGIT2_MIN_VER = v"0.23.0" @test sig3.email == sig.email #end +#@testset "URL parsing" begin + # Use all named group + m = match(LibGit2.URL_REGEX, "https://user:pass@hostname.com:80/path/repo.git") + @test m[:scheme] == "https" + @test m[:user] == "user" + @test m[:password] == "pass" + @test m[:host] == "hostname.com" + @test m[:port] == "80" + @test m[:path] == "/path/repo.git" + + # Realistic example HTTP example + m = match(LibGit2.URL_REGEX, "https://github.com/JuliaLang/Example.jl.git") + @test m[:scheme] == "https" + @test m[:user] == nothing + @test m[:password] == nothing + @test m[:host] == "github.com" + @test m[:port] == nothing + @test m[:path] == "/JuliaLang/Example.jl.git" + + # Realistic example SSH example + m = match(LibGit2.URL_REGEX, "git@github.com:JuliaLang/Example.jl.git") + @test m[:scheme] == nothing + @test m[:user] == "git" + @test m[:password] == nothing + @test m[:host] == "github.com" + @test m[:port] == nothing + @test m[:path] == "JuliaLang/Example.jl.git" + + # Make sure that a username can contain special characters + m = match(LibGit2.URL_REGEX, "user-name@hostname.com") + @test m[:user] == "user-name" +#end + mktempdir() do dir # test parameters repo_url = "https://github.com/JuliaLang/Example.jl" From a48ce1697af849548a2c2c0e065635e357401ac2 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 16 Aug 2016 15:13:07 -0500 Subject: [PATCH 069/114] Correct invalid SSH URL An SSH URL with a path uses a colon between the host and the path: git@github.com:JuliaLang/Example.jl (correct) git@github.com/JuliaLang/Example.jl (incorrect) (cherry picked from commit 749d099e13f44d516c0916e9c9d02c917f96edc9) ref #18066 --- test/libgit2-online.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/libgit2-online.jl b/test/libgit2-online.jl index 436003eb37160..61b683f087138 100644 --- a/test/libgit2-online.jl +++ b/test/libgit2-online.jl @@ -7,13 +7,12 @@ ######### # init & clone mktempdir() do dir - repo_url = "github.com/JuliaLang/Example.jl" - https_prefix = "https://" - ssh_prefix = "git@" + repo_url_https = "https://github.com/JuliaLang/Example.jl" + repo_url_git = "git@github.com:JuliaLang/Example.jl" #@testset "Cloning repository" begin #@testset "with 'https' protocol" begin repo_path = joinpath(dir, "Example1") - repo = LibGit2.clone(https_prefix*repo_url, repo_path) + repo = LibGit2.clone(repo_url_https, repo_path) try @test isdir(repo_path) @test isdir(joinpath(repo_path, ".git")) @@ -27,7 +26,7 @@ mktempdir() do dir repo_path = joinpath(dir, "Example2") # credentials are required because github tries to authenticate on unknown repo cred = LibGit2.UserPasswordCredentials("","") # empty credentials cause authentication error - LibGit2.clone(https_prefix*repo_url*randstring(10), repo_path, payload=Nullable(cred)) + LibGit2.clone(repo_url_https*randstring(10), repo_path, payload=Nullable(cred)) error("unexpected") catch ex @test isa(ex, LibGit2.Error.GitError) @@ -37,12 +36,13 @@ mktempdir() do dir #TODO: remove or condition on libgit2 features this test when ssh protocol will be supported #@testset "with 'ssh' protocol (by default is not supported)" begin + repo_path = joinpath(dir, "Example3") + repo = LibGit2.clone(repo_url_git, repo_path) try - repo_path = joinpath(dir, "Example3") - @test_throws LibGit2.Error.GitError LibGit2.clone(ssh_prefix*repo_url, repo_path) - catch ex - # but we cloned succesfully, so check that repo was created - ex.fail == 1 && @test isdir(joinpath(path, ".git")) + @test isdir(repo_path) + @test isdir(joinpath(repo_path, ".git")) + finally + finalize(repo) end #end #end From e7d3848d3bed27b9ea62210cff40d0feeeec57a2 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 17 Aug 2016 13:24:57 -0500 Subject: [PATCH 070/114] Removed SSH test from libgit2-online Test was only working for environments which had GitHub SSH keys setup. (cherry picked from commit 1e98a5ab2a230caac320f69b1d801bccd95c344f) ref #18066 --- test/libgit2-online.jl | 19 +++---------------- test/libgit2.jl | 1 - 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/test/libgit2-online.jl b/test/libgit2-online.jl index 61b683f087138..dd02d6c42d37f 100644 --- a/test/libgit2-online.jl +++ b/test/libgit2-online.jl @@ -7,12 +7,11 @@ ######### # init & clone mktempdir() do dir - repo_url_https = "https://github.com/JuliaLang/Example.jl" - repo_url_git = "git@github.com:JuliaLang/Example.jl" + repo_url = "https://github.com/JuliaLang/Example.jl" #@testset "Cloning repository" begin #@testset "with 'https' protocol" begin repo_path = joinpath(dir, "Example1") - repo = LibGit2.clone(repo_url_https, repo_path) + repo = LibGit2.clone(repo_url, repo_path) try @test isdir(repo_path) @test isdir(joinpath(repo_path, ".git")) @@ -26,25 +25,13 @@ mktempdir() do dir repo_path = joinpath(dir, "Example2") # credentials are required because github tries to authenticate on unknown repo cred = LibGit2.UserPasswordCredentials("","") # empty credentials cause authentication error - LibGit2.clone(repo_url_https*randstring(10), repo_path, payload=Nullable(cred)) + LibGit2.clone(repo_url*randstring(10), repo_path, payload=Nullable(cred)) error("unexpected") catch ex @test isa(ex, LibGit2.Error.GitError) @test ex.code == LibGit2.Error.EAUTH end #end - - #TODO: remove or condition on libgit2 features this test when ssh protocol will be supported - #@testset "with 'ssh' protocol (by default is not supported)" begin - repo_path = joinpath(dir, "Example3") - repo = LibGit2.clone(repo_url_git, repo_path) - try - @test isdir(repo_path) - @test isdir(joinpath(repo_path, ".git")) - finally - finalize(repo) - end - #end #end end diff --git a/test/libgit2.jl b/test/libgit2.jl index 64c7fa7f0e47a..5c24612edada6 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -112,7 +112,6 @@ const LIBGIT2_MIN_VER = v"0.23.0" mktempdir() do dir # test parameters repo_url = "https://github.com/JuliaLang/Example.jl" - ssh_prefix = "git@" cache_repo = joinpath(dir, "Example") test_repo = joinpath(dir, "Example.Test") test_sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time(), 0), 0) From f703e65f5e442a7b221c1289f7c9eeca4a695293 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 17 Aug 2016 13:40:45 -0500 Subject: [PATCH 071/114] Update URL parsing tests Added a new explicit SSH protocol test which generated other changes. (cherry picked from commit ea8fa251fdb7434e2e84fab58d74d5079412bf78) ref #18066 --- test/libgit2.jl | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/test/libgit2.jl b/test/libgit2.jl index 5c24612edada6..4597c57e03465 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -77,16 +77,34 @@ const LIBGIT2_MIN_VER = v"0.23.0" #end #@testset "URL parsing" begin - # Use all named group - m = match(LibGit2.URL_REGEX, "https://user:pass@hostname.com:80/path/repo.git") + # HTTPS URL + m = match(LibGit2.URL_REGEX, "https://user:pass@server.com:80/org/project.git") @test m[:scheme] == "https" @test m[:user] == "user" @test m[:password] == "pass" - @test m[:host] == "hostname.com" + @test m[:host] == "server.com" @test m[:port] == "80" - @test m[:path] == "/path/repo.git" + @test m[:path] == "/org/project.git" - # Realistic example HTTP example + # SSH URL + m = match(LibGit2.URL_REGEX, "ssh://user:pass@server:22/project.git") + @test m[:scheme] == "ssh" + @test m[:user] == "user" + @test m[:password] == "pass" + @test m[:host] == "server" + @test m[:port] == "22" + @test m[:path] == "/project.git" + + # SSH URL using scp-like syntax + m = match(LibGit2.URL_REGEX, "user@server:project.git") + @test m[:scheme] == nothing + @test m[:user] == "user" + @test m[:password] == nothing + @test m[:host] == "server" + @test m[:port] == nothing + @test m[:path] == "project.git" + + # Realistic example from GitHub using HTTPS m = match(LibGit2.URL_REGEX, "https://github.com/JuliaLang/Example.jl.git") @test m[:scheme] == "https" @test m[:user] == nothing @@ -95,7 +113,7 @@ const LIBGIT2_MIN_VER = v"0.23.0" @test m[:port] == nothing @test m[:path] == "/JuliaLang/Example.jl.git" - # Realistic example SSH example + # Realistic example from GitHub using SSH m = match(LibGit2.URL_REGEX, "git@github.com:JuliaLang/Example.jl.git") @test m[:scheme] == nothing @test m[:user] == "git" @@ -104,7 +122,7 @@ const LIBGIT2_MIN_VER = v"0.23.0" @test m[:port] == nothing @test m[:path] == "JuliaLang/Example.jl.git" - # Make sure that a username can contain special characters + # Make sure usernames can contain special characters m = match(LibGit2.URL_REGEX, "user-name@hostname.com") @test m[:user] == "user-name" #end From 5920a4710ea1da3b13b337eadf1e54ce89b1e662 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 18 Aug 2016 05:49:20 +0000 Subject: [PATCH 072/114] Fix compilation on llvm 3.9 (cherry picked from commit 090e9e435319b8c422f1ac9ef29fad510e45b509) ref #18104 --- src/debuginfo.cpp | 79 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index c7e598fc5de4e..89e4e5d4a6859 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -29,14 +29,10 @@ #else #include #endif -#ifdef _OS_DARWIN_ #include -#endif -#ifdef _OS_WINDOWS_ #include -# ifdef LLVM37 -# include -# endif +#ifdef LLVM37 +# include #endif #if defined(USE_MCJIT) && !defined(LLVM36) && defined(_OS_DARWIN_) @@ -896,23 +892,52 @@ calc_gnu_debuglink_crc32(const void *buf, size_t size) crc = g_crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); return crc ^ ~0U; } -static ErrorOr> openDebugInfo(StringRef debuginfopath, const debug_link_info &info) + +template +static inline void ignoreError(T &err) +{ +#if defined(LLVM39) && !defined(NDEBUG) + consumeError(err.takeError()); +#endif +} + +#ifdef LLVM39 +static Expected> +#else +static ErrorOr> +#endif +openDebugInfo(StringRef debuginfopath, const debug_link_info &info) { auto SplitFile = MemoryBuffer::getFile(debuginfopath); - if (std::error_code EC = SplitFile.getError()) + if (std::error_code EC = SplitFile.getError()) { +#ifdef LLVM39 + return errorCodeToError(EC); +#else return EC; +#endif + } uint32_t crc32 = calc_gnu_debuglink_crc32( SplitFile.get()->getBufferStart(), SplitFile.get()->getBufferSize()); - if (crc32 != info.crc32) + if (crc32 != info.crc32) { +#ifdef LLVM39 + return errorCodeToError(object::object_error::arch_not_found); +#else return object::object_error::arch_not_found; +#endif + } auto error_splitobj = object::ObjectFile::createObjectFile( SplitFile.get().get()->getMemBufferRef(), sys::fs::file_magic::unknown); - if (std::error_code EC = error_splitobj.getError()) - return EC; + if (!error_splitobj) { +#ifdef LLVM39 + return error_splitobj.takeError(); +#else + return error_splitobj.getError(); +#endif + } // successfully validated and loaded split debug info file return object::OwningBinary( @@ -1139,32 +1164,44 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, debug_link_info info = getDebuglink(*debugobj); if (!info.filename.empty()) { size_t sep = fname.rfind('/'); - ErrorOr> DebugInfo(std::errc::no_such_file_or_directory); +#ifdef LLVM39 + Expected> + DebugInfo(errorCodeToError(std::make_error_code(std::errc::no_such_file_or_directory))); + // Can't find a way to construct an empty Expected object + // that can be ignored. + ignoreError(DebugInfo); +#else + ErrorOr> + DebugInfo(std::errc::no_such_file_or_directory); +#endif if (fname.substr(sep + 1) != info.filename) { debuginfopath = fname.substr(0, sep + 1); debuginfopath += info.filename; DebugInfo = openDebugInfo(debuginfopath, info); } - if (DebugInfo.getError()) { + if (!DebugInfo) { debuginfopath = fname.substr(0, sep + 1); debuginfopath += ".debug/"; debuginfopath += info.filename; + ignoreError(DebugInfo); DebugInfo = openDebugInfo(debuginfopath, info); } - if (DebugInfo.getError()) { + if (!DebugInfo) { debuginfopath = "/usr/lib/debug/"; debuginfopath += fname.substr(0, sep + 1); debuginfopath += info.filename; + ignoreError(DebugInfo); DebugInfo = openDebugInfo(debuginfopath, info); } - if (DebugInfo.getError()) { - // no split debug info found - // should we warn the user? - } - else { + if (DebugInfo) { errorobj = std::move(DebugInfo); + // Yes, we've checked, and yes LLVM want us to check again. + assert(errorobj); debugobj = errorobj->getBinary(); } + else { + ignoreError(DebugInfo); + } } #endif } @@ -1224,13 +1261,11 @@ bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, errorobj.release(); #endif } -#ifdef LLVM39 else { // TODO: report the error instead of silently consuming it? // jl_error might run into the same error again... - consumeError(errorobj.takeError()); + ignoreError(errorobj); } -#endif // update cache objfileentry_t entry = {*obj, *context, *slide, *section_slide}; From 738f5caf5056f219b22967f1826906576d4ca652 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Thu, 18 Aug 2016 09:16:24 +0530 Subject: [PATCH 073/114] Build curl with minimal external libraries. Link to our own libssh2. (cherry picked from commit 4e12052db87022f46443f8c0dac08a8824a5f07c) ref #18103 --- deps/curl.mk | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/deps/curl.mk b/deps/curl.mk index b8bf9ccbd8fec..060b41b7969a1 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -18,7 +18,14 @@ $(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VE $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure mkdir -p $(dir $@) cd $(dir $@) && \ - $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) --without-ssl --with-mbedtls=$(build_prefix) CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) \ + --without-ssl --without-gnutls --without-gssapi \ + --without-libidn --without-libmetalink --without-librtmp \ + --without-nghttp2 --without-nss --without-polarssl \ + --without-spnego --disable-ares --disable-ldap \ + --disable-ldaps --without-zsh-functions-dir \ + --with-libssh2=$(build_prefix) --with-mbedtls=$(build_prefix) \ + CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" touch -c $@ $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status From 01c76ea366710dd5dba36f13b966a0cae8af7022 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Thu, 18 Aug 2016 10:39:39 +0800 Subject: [PATCH 074/114] Improve arch/cpu detection/selection on ARM and AArch64 * Allow `cpu_target` to specify a generic arch, matching the behavior on x86 * Detect the CPU arch version with `uname` * Require `armv6` Close #13270 (`armv5` is not supported) Fix #18042 * Remove warning about generic arch since it's not really useful Fix #17549 * Require at least the same ARM arch version and profile the C code is compiled with (cherry picked from commit 760bc41d6410bb0b398782f1da9353c4c31869a6) ref #18100 --- README.arm.md | 7 ++- src/codegen.cpp | 130 +++++++++++++++++++++++++++++++++++++++++++----- src/disasm.cpp | 4 +- 3 files changed, 125 insertions(+), 16 deletions(-) diff --git a/README.arm.md b/README.arm.md index 1c50538172cb9..c9c823cd68d4f 100644 --- a/README.arm.md +++ b/README.arm.md @@ -1,7 +1,12 @@ # Julia binaries for ARM [Nightly builds](https://status.julialang.org/download/linux-arm) are -available for ARM. +available for ARMv7-A. + +# Hardware requirements + +Julia requires at least `armv6` and `vfpv2` instruction sets. It's recommanded +to use at least `armv7-a`. `armv5` or soft float are not supported. # Building Julia on ARM diff --git a/src/codegen.cpp b/src/codegen.cpp index 55b8aafba0118..e7d07688e9dad 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -87,6 +87,7 @@ #if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) # include +# include #endif #if defined(USE_POLLY) #include @@ -5560,10 +5561,68 @@ static void init_julia_llvm_env(Module *m) addOptimizationPasses(jl_globalPM); } +static inline std::string getNativeTarget() +{ + std::string cpu = sys::getHostCPUName(); +#if defined(_CPU_ARM_) + // Try slightly harder than LLVM at determine the CPU architecture. + if (cpu == "generic") { + // This is the most reliable way I can find + // `/proc/cpuinfo` changes between kernel versions + struct utsname name; + if (uname(&name) >= 0) { + // name.machine is the elf_platform in the kernel. + if (strcmp(name.machine, "armv6l") == 0) { + return "armv6"; + } + if (strcmp(name.machine, "armv7l") == 0) { + return "armv7"; + } + if (strcmp(name.machine, "armv7ml") == 0) { + // Thumb + return "armv7-m"; + } + if (strcmp(name.machine, "armv8l") == 0 || + strcmp(name.machine, "aarch64") == 0) { + return "armv8"; + } + } + } +#endif + return cpu; +} + +#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) +// Check if the cpu name is a ARM/AArch64 arch name and return a +// string that can be used as LLVM feature name +static inline std::string checkARMArchFeature(const std::string &cpu) +{ + const char *prefix = "armv"; + size_t prefix_len = strlen(prefix); + if (cpu.size() <= prefix_len || + memcmp(cpu.data(), prefix, prefix_len) != 0 || + cpu[prefix_len] < '1' || cpu[prefix_len] > '9') + return std::string(); +#if defined(_CPU_ARM_) + // "v7" and "v8" are not available in the form of `armv*` + // in the feature list + if (cpu == "armv7") { + return "v7"; + } + else if (cpu == "armv8") { + return "v8"; + } + return cpu; +#else + return cpu.substr(3); +#endif +} +#endif + // Helper to figure out what features to set for the LLVM target // If the user specifies native (or does not specify) we default // using the API provided by LLVM -static inline SmallVector getTargetFeatures() +static inline SmallVector getTargetFeatures(std::string &cpu) { StringMap HostFeatures; if (!strcmp(jl_options.cpu_target,"native")) { @@ -5592,16 +5651,63 @@ static inline SmallVector getTargetFeatures() #endif // Figure out if we know the cpu_target - std::string cpu = strcmp(jl_options.cpu_target,"native") ? jl_options.cpu_target : sys::getHostCPUName(); - if (cpu.empty() || cpu == "generic") { - jl_printf(JL_STDERR, "WARNING: unable to determine host cpu name.\n"); -#if defined(_CPU_ARM_) && defined(__ARM_PCS_VFP) - // Check if this is required when you have read the features directly from the processor - // This affects the platform calling convention. - // TODO: enable vfp3 for ARMv7+ (but adapt the ABI) - HostFeatures["vfp2"] = true; -#endif + cpu = (strcmp(jl_options.cpu_target,"native") ? jl_options.cpu_target : + getNativeTarget()); +#if defined(_CPU_ARM_) + // Figure out what we are compiling against from the C defines. + // This might affect ABI but is fine since + // 1. We define the C ABI explicitly. + // 2. This does not change when running the same binary on different + // machines. + // This shouldn't affect making generic binaries since that requires a + // generic C -march anyway. + HostFeatures["vfp2"] = true; + + // Arch version +#if __ARM_ARCH >= 8 + HostFeatures["v8"] = true; +#elif __ARM_ARCH >= 7 + HostFeatures["v7"] = true; +#else + // minimum requirement + HostFeatures["v6"] = true; +#endif + + // ARM profile + // Only do this on ARM and not AArch64 since LLVM aarch64 backend + // doesn't support setting profiles. + // AFAIK there's currently no 64bit R and M profile either + // (v8r and v8m are both 32bit) +#if defined(__ARM_ARCH_PROFILE) +# if __ARM_ARCH_PROFILE == 'A' + HostFeatures["aclass"] = true; +# elif __ARM_ARCH_PROFILE == 'R' + HostFeatures["rclass"] = true; +# elif __ARM_ARCH_PROFILE == 'M' + // Thumb + HostFeatures["mclass"] = true; +# endif +#endif +#endif // _CPU_ARM_ + + // On ARM and AArch64, allow using cpu_target to specify a CPU architecture + // which is specified in the feature set in LLVM. +#if defined(_CPU_ARM_) || defined(_CPU_AARCH64_) + // Supported ARM arch names on LLVM 3.8: + // armv6, armv6-m, armv6j, armv6k, armv6kz, armv6s-m, armv6t2, + // armv7, armv7-a, armv7-m, armv7-r, armv7e-m, armv7k, armv7s, + // armv8, armv8-a, armv8.1-a, armv8.2-a + // Additional ARM arch names on LLVM 3.9: + // armv8-m.base, armv8-m.main + // + // Supported AArch64 arch names on LLVM 3.8: + // armv8.1a, armv8.2a + std::string arm_arch = checkARMArchFeature(cpu); + if (!arm_arch.empty()) { + HostFeatures[arm_arch] = true; + cpu = "generic"; } +#endif SmallVector attr; for (StringMap::const_iterator it = HostFeatures.begin(); it != HostFeatures.end(); it++) { @@ -5718,8 +5824,8 @@ extern "C" void jl_init_codegen(void) TheTriple.setEnvironment(Triple::ELF); #endif #endif - std::string TheCPU = strcmp(jl_options.cpu_target,"native") ? jl_options.cpu_target : sys::getHostCPUName(); - SmallVector targetFeatures = getTargetFeatures( ); + std::string TheCPU; + SmallVector targetFeatures = getTargetFeatures(TheCPU); jl_TargetMachine = eb.selectTarget( TheTriple, "", diff --git a/src/disasm.cpp b/src/disasm.cpp index df3f15604e11b..356e57add0b9b 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -370,9 +370,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, { // GC safe // Get the host information - std::string TripleName; - if (TripleName.empty()) - TripleName = sys::getDefaultTargetTriple(); + std::string TripleName = sys::getDefaultTargetTriple(); Triple TheTriple(Triple::normalize(TripleName)); std::string MCPU = sys::getHostCPUName(); From 22a73dd4c2e1e9cf6260190776e683342f04e938 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Thu, 18 Aug 2016 04:23:29 -0700 Subject: [PATCH 075/114] fix "recommanded" typo (cherry picked from commit 59771677ce0df251a6e105a94d961918f0cfbef8) ref #18100 --- README.arm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.arm.md b/README.arm.md index c9c823cd68d4f..f31eed9d51ef3 100644 --- a/README.arm.md +++ b/README.arm.md @@ -5,7 +5,7 @@ available for ARMv7-A. # Hardware requirements -Julia requires at least `armv6` and `vfpv2` instruction sets. It's recommanded +Julia requires at least `armv6` and `vfpv2` instruction sets. It's recommended to use at least `armv7-a`. `armv5` or soft float are not supported. # Building Julia on ARM From 962a2cc2555c4408ea5e1e36d5fe1cf54a98682f Mon Sep 17 00:00:00 2001 From: Andreas Noack Date: Thu, 18 Aug 2016 18:24:33 -0400 Subject: [PATCH 076/114] Add istriu, istril, issymmetric, and ishermitian methods for UniformScaling (#17950) (cherry picked from commit 57df5f15dff5b30a9e34308b4162fa2b4ca4a5d4) --- base/linalg/uniformscaling.jl | 5 +++++ test/linalg/uniformscaling.jl | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/base/linalg/uniformscaling.jl b/base/linalg/uniformscaling.jl index 227e41ac5dc0c..91098a08da173 100644 --- a/base/linalg/uniformscaling.jl +++ b/base/linalg/uniformscaling.jl @@ -29,6 +29,11 @@ one{T}(J::UniformScaling{T}) = one(UniformScaling{T}) zero{T}(::Type{UniformScaling{T}}) = UniformScaling(zero(T)) zero{T}(J::UniformScaling{T}) = zero(UniformScaling{T}) +istriu(::UniformScaling) = true +istril(::UniformScaling) = true +issymmetric(::UniformScaling) = true +ishermitian(J::UniformScaling) = isreal(J.λ) + (+)(J1::UniformScaling, J2::UniformScaling) = UniformScaling(J1.λ+J2.λ) (+){T}(B::BitArray{2},J::UniformScaling{T}) = Array(B) + J (+)(J::UniformScaling, B::BitArray{2}) = J + Array(B) diff --git a/test/linalg/uniformscaling.jl b/test/linalg/uniformscaling.jl index f20dd0315cab8..477306c6fc6ba 100644 --- a/test/linalg/uniformscaling.jl +++ b/test/linalg/uniformscaling.jl @@ -16,6 +16,12 @@ srand(123) @test one(UniformScaling(rand(Complex128))) == one(UniformScaling{Complex128}) @test eltype(one(UniformScaling(rand(Complex128)))) == Complex128 @test -one(UniformScaling(2)) == UniformScaling(-1) +@test istriu(I) +@test istril(I) +@test issymmetric(I) +@test issymmetric(UniformScaling(complex(1.0,1.0))) +@test ishermitian(I) +@test !ishermitian(UniformScaling(complex(1.0,1.0))) α = randn() @test α .* UniformScaling(1.0) == UniformScaling(1.0) .* α From 6d0aa14594b83ffd500a4dadac52f048a249891f Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 18 Aug 2016 12:21:55 -0400 Subject: [PATCH 077/114] fix #18085, segfault on method add in loop Closures that are part of out-of-scope method adds were not lifted out of top-level loops. (cherry picked from commit c9c7bfa2146ff9a613057e2513b088dc9dc9b495) ref #18119 --- src/julia-syntax.scm | 2 +- test/core.jl | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 2e483095e1952..3c37a6a9f8759 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2871,7 +2871,7 @@ f(x) = yt(x) (let* ((exprs (lift-toplevel (convert-lambda lam2 '|#anon| #t '()))) (top-stmts (cdr exprs)) (newlam (renumber-slots-and-labels (linearize (car exprs))))) - `(block + `(toplevel-butlast ,@top-stmts ,@sp-inits (method ,name ,(cl-convert sig fname lam namemap toplevel interp) diff --git a/test/core.jl b/test/core.jl index ee4605ff60704..0c39246b57bab 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4469,3 +4469,11 @@ let k(x) = (k = x; k) @test k(1) == 1 end + +# issue #18085 +f18085(a,x...) = (0,) +for (f,g) in ((:asin,:sin), (:acos,:cos)) + gx = eval(g) + f18085(::Type{Val{f}},x...) = map(x->2gx(x), f18085(Val{g},x...)) +end +@test f18085(Val{:asin},3) === (0.0,) From 51aadfec4f0dd51e986608ef9057674c3d07f573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Biernat?= Date: Fri, 19 Aug 2016 11:05:36 +0200 Subject: [PATCH 078/114] rational powers (fixes #18114) (#18118) Fixes precision problem for rational powers of BigFloats (#18114) (cherry picked from commit db03824a50d3469848b2e56ca187e59fb0376488) --- base/rational.jl | 4 ++-- test/numbers.jl | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/base/rational.jl b/base/rational.jl index 067b8b570deac..6ff823701088d 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -375,8 +375,8 @@ function ^(x::Rational, n::Integer) end ^(x::Number, y::Rational) = x^(y.num/y.den) -^{T<:AbstractFloat}(x::T, y::Rational) = x^(convert(T, y.num / y.den)) -^{T<:AbstractFloat}(x::Complex{T}, y::Rational) = x^(convert(T, y.num / y.den)) +^{T<:AbstractFloat}(x::T, y::Rational) = x^convert(T,y) +^{T<:AbstractFloat}(x::Complex{T}, y::Rational) = x^convert(T,y) ^{T<:Rational}(z::Complex{T}, n::Bool) = n ? z : one(z) # to resolve ambiguity function ^{T<:Rational}(z::Complex{T}, n::Integer) diff --git a/test/numbers.jl b/test/numbers.jl index ae834f109a376..5c5f322e265f5 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2139,7 +2139,8 @@ rationalize(nextfloat(0.0)) == 0//1 # rational-exponent promotion rules (issue #3155): @test 2.0f0^(1//3) == 2.0f0^(1.0f0/3) @test 2^(1//3) == 2^(1/3) - +# no loss of precision for rational powers (issue #18114) +@test BigFloat(2)^(BigFloat(1)/BigFloat(3)) == BigFloat(2)^(1//3) # large shift amounts @test Int32(-1)>>31 == -1 From ac9627ce208d61b45ec880debb2696d41bdef7eb Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Fri, 19 Aug 2016 05:11:45 -0700 Subject: [PATCH 079/114] More tests for osnames (#18121) (cherry picked from commit 9fd79eff79608dc189c6e7fbb38232d386afe730) --- test/misc.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/misc.jl b/test/misc.jl index d10b46aafa732..bf3795ffb8051 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -206,6 +206,15 @@ end @test isa(ex, ErrorException) && ex.msg == "cannot assign variables in other modules" end +@test !Base.is_unix(:Windows) +@test !Base.is_linux(:Windows) +@test Base.is_linux(:Linux) +@test Base.is_windows(:Windows) +@test Base.is_windows(:NT) +@test !Base.is_windows(:Darwin) +@test Base.is_apple(:Darwin) +@test Base.is_apple(:Apple) +@test !Base.is_apple(:Windows) @test Base.is_unix(:Darwin) @test Base.is_unix(:FreeBSD) @test_throws ArgumentError Base.is_unix(:BeOS) From 6dfbbfd4608a96095d1191a9a540d8bd7810372c Mon Sep 17 00:00:00 2001 From: Katharine Hyatt Date: Fri, 19 Aug 2016 05:12:33 -0700 Subject: [PATCH 080/114] Two missing tests for intset (#18128) (cherry picked from commit a08bfcd8eae69bc0ff03d74ba2df986def6346c6) --- test/intset.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/intset.jl b/test/intset.jl index 5ebd531a46e09..ff062ba16c859 100644 --- a/test/intset.jl +++ b/test/intset.jl @@ -14,6 +14,7 @@ data_out = collect(s) # eltype, similar @test is(eltype(IntSet()), Int64) +@test is(eltype(IntSet), Int64) @test isequal(similar(IntSet([1,2,3])), IntSet()) # show @@ -33,6 +34,8 @@ s = IntSet([1,2,10,20,200,300,1000,10000,10002]) @test !in(1,s) @test !in(10002,s) @test in(10000,s) +@test in(10000.0,s) +@test !in(10002.0,s) @test_throws ArgumentError first(IntSet()) @test_throws ArgumentError last(IntSet()) t = copy(s) From a61909d5af2c95ac5cd2411bdd43ee49d95e216d Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 19 Aug 2016 07:19:58 -0500 Subject: [PATCH 081/114] Make similar faster and safer (#18107) * Make similar faster and safer By converting the eltype argument into a ::Type{T} parameter, we avoid runtime method lookup. Perhaps more importantly, the introduction of NeedsShaping makes `similar` safer, by preventing any possibility of an infinite recursion. * Test that similar throws a MethodError for unsupported dims types (cherry picked from commit 66bacece27a1c34fcca7a92c0ba496d5c444dce7) --- base/abstractarray.jl | 15 ++++++++------- test/offsetarray.jl | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 2a292343bc9d0..045d255ba91b3 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -9,6 +9,7 @@ typealias RangeIndex Union{Int, Range{Int}, AbstractUnitRange{Int}, Colon} typealias DimOrInd Union{Integer, AbstractUnitRange} typealias IntOrInd Union{Int, AbstractUnitRange} typealias DimsOrInds{N} NTuple{N,DimOrInd} +typealias NeedsShaping Union{Tuple{Integer,Vararg{Integer}}, Tuple{OneTo,Vararg{OneTo}}} macro _inline_pure_meta() Expr(:meta, :inline, :pure) @@ -413,14 +414,14 @@ different element type it will create a regular `Array` instead: 2.18425e-314 2.18425e-314 2.18425e-314 2.18425e-314 """ -similar{T}(a::AbstractArray{T}) = similar(a, T) -similar( a::AbstractArray, T::Type) = similar(a, T, to_shape(indices(a))) -similar{T}(a::AbstractArray{T}, dims::Tuple) = similar(a, T, to_shape(dims)) -similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, to_shape(dims)) -similar( a::AbstractArray, T::Type, dims::DimOrInd...) = similar(a, T, to_shape(dims)) -similar( a::AbstractArray, T::Type, dims) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray{T}) = similar(a, T) +similar{T}(a::AbstractArray, ::Type{T}) = similar(a, T, to_shape(indices(a))) +similar{T}(a::AbstractArray{T}, dims::Tuple) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray{T}, dims::DimOrInd...) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray, ::Type{T}, dims::DimOrInd...) = similar(a, T, to_shape(dims)) +similar{T}(a::AbstractArray, ::Type{T}, dims::NeedsShaping) = similar(a, T, to_shape(dims)) # similar creates an Array by default -similar{N}(a::AbstractArray, T::Type, dims::Dims{N}) = Array{T,N}(dims) +similar{T,N}(a::AbstractArray, ::Type{T}, dims::Dims{N}) = Array{T,N}(dims) to_shape(::Tuple{}) = () to_shape(dims::Dims) = dims diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 4f4e70933a55d..c2ba9687c4ccc 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -369,3 +369,20 @@ for s = -5:5 end end # let + +# Check that similar throws a MethodError rather than a +# StackOverflowError if no appropriate method has been defined +# (#18107) +module SimilarUR + using Base.Test + immutable MyURange <: AbstractUnitRange{Int} + start::Int + stop::Int + end + ur = MyURange(1,3) + a = Array{Int}(2) + @test_throws MethodError similar(a, ur) + @test_throws MethodError similar(a, Float64, ur) + @test_throws MethodError similar(a, Float64, (ur,)) + @test_throws MethodError similar(a, (2.0,3.0)) +end From fd5bfed331634e244bb77364cd4d48cd87939c8b Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Fri, 19 Aug 2016 14:56:23 +0200 Subject: [PATCH 082/114] Fix incorrect code block in rounding modes docs (#18137) The `::` syntax appearing before the `doctest::` block was causing the doctest not be rendered correctly. (cherry picked from commit ec5283719fe714876fa7b9c67ad6eab240a902d1) --- doc/manual/integers-and-floating-point-numbers.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/integers-and-floating-point-numbers.rst b/doc/manual/integers-and-floating-point-numbers.rst index 2d72791d49a8e..136e5e92e463f 100644 --- a/doc/manual/integers-and-floating-point-numbers.rst +++ b/doc/manual/integers-and-floating-point-numbers.rst @@ -505,11 +505,10 @@ Rounding modes If a number doesn't have an exact floating-point representation, it must be rounded to an appropriate representable value, however, if wanted, the manner in which this rounding is done can be changed according to the rounding modes -presented in the `IEEE 754 standard `_:: +presented in the `IEEE 754 standard `_. .. doctest:: - julia> x = 1.1; y = 0.1; julia> x + y From ca5c823bb34fd4dbd511aa8b022f5b52d177fe06 Mon Sep 17 00:00:00 2001 From: "Steven G. Johnson" Date: Fri, 19 Aug 2016 14:32:12 -0400 Subject: [PATCH 083/114] make sure .= uses Base.identity, not a local identity (#18122) (cherry picked from commit b117e59815813e90bf6953190c25c2c02f79db05) --- src/julia-syntax.scm | 2 +- test/broadcast.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 3c37a6a9f8759..f762753a5f80f 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1695,7 +1695,7 @@ (expand-forms `(call (top broadcast!) ,(from-lambda (cadr e)) ,lhs-view ,@(caddr e)))) (if (null? lhs) (expand-forms e) - (expand-forms `(call (top broadcast!) identity ,lhs-view ,e)))))) + (expand-forms `(call (top broadcast!) (top identity) ,lhs-view ,e)))))) ;; table mapping expression head to a function expanding that form (define expand-table diff --git a/test/broadcast.jl b/test/broadcast.jl index 7cc1ceffa500f..dfbbeed1c8950 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -284,6 +284,10 @@ let d = Dict(:foo => [1,3,7], (3,4) => [5,9]) d[3,4] .-= 1 @test d[3,4] == [4,8] end +let identity = error, x = [1,2,3] + x .= 1 # make sure it goes to broadcast!(Base.identity, ...), not identity + @test x == [1,1,1] +end # PR 16988 @test Base.promote_op(+, Bool) === Int From bd679a346bbb79925c8f6b7f05ca31e83616057b Mon Sep 17 00:00:00 2001 From: Katie Hyatt Date: Thu, 18 Aug 2016 18:04:47 -0700 Subject: [PATCH 084/114] Missing tests for intfuncs (cherry picked from commit 9df2cb914fcdf770d064ed1d9c8118ad056fce5d) ref #18132 --- test/intfuncs.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/intfuncs.jl b/test/intfuncs.jl index 7a6c84c54670e..822bad6805a07 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -2,6 +2,7 @@ # Int32 and Int64 take different code paths -- test both for T in (Int32, Int64) + @test gcd(T(3)) === T(3) @test gcd(T(3), T(5)) === T(1) @test gcd(T(3), T(15)) === T(3) @test gcd(T(0), T(15)) === T(15) @@ -16,6 +17,7 @@ for T in (Int32, Int64) @test gcd(typemin(T), T(1)) === T(1) @test_throws OverflowError gcd(typemin(T), typemin(T)) + @test lcm(T(2)) === T(2) @test lcm(T(2), T(3)) === T(6) @test lcm(T(4), T(6)) === T(12) @test lcm(T(3), T(0)) === T(0) @@ -86,6 +88,8 @@ let n = rand(Int) @test ndigits(n) == ndigits(big(n)) == ndigits(n, 10) end +@test bin('3') == "110011" +@test bin('3',7) == "0110011" @test bin(3) == "11" @test bin(3, 2) == "11" @test bin(3, 3) == "011" @@ -103,8 +107,11 @@ end @test base(2, 5, 7) == "0000101" +@test bits(Int16(3)) == "0000000000000011" +@test bits('3') == "00000000000000000000000000110011" @test bits(1035) == (Int == Int32 ? "00000000000000000000010000001011" : "0000000000000000000000000000000000000000000000000000010000001011") +@test bits(Int128(3)) == "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000011" @test digits(4, 2) == [0, 0, 1] @test digits(5, 3) == [2, 1] @@ -116,6 +123,9 @@ end @test count_zeros(Int64(1)) == 63 +@test factorial(3) == 6 +@test_throws DomainError factorial(-3) + @test isqrt(4) == 2 @test isqrt(5) == 2 # issue #4884 From 782cf362d31a0aeac9f0d02cf79b7deab1096373 Mon Sep 17 00:00:00 2001 From: Katie Hyatt Date: Thu, 18 Aug 2016 19:39:52 -0700 Subject: [PATCH 085/114] Tests for missing abstract array utils (cherry picked from commit 94c62eb41c95fde9a7cbe95342507936ebb35105) ref #18133 --- test/abstractarray.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index b0f3ae07526ae..ab9f68f53ef2e 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -722,3 +722,8 @@ end #flipdim on empty @test flipdim(Diagonal([]),1) == Diagonal([]) + +# ndims and friends +@test ndims(Diagonal(rand(1:5,5))) == 2 +@test ndims(Diagonal{Float64}) == 2 +@test Base.elsize(Diagonal(rand(1:5,5))) == sizeof(Int) From 09f7005d48e3390333cfd2d381b6b34aa5a5bece Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 18 Aug 2016 19:25:16 -0400 Subject: [PATCH 086/114] fix #18129, optimize some more cases of captured variables (cherry picked from commit 01378992c843d9aa99b2edddd19d387ffe65a673) ref #18130 --- src/julia-syntax.scm | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index f762753a5f80f..aaf07bca7d74a 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2721,25 +2721,27 @@ f(x) = yt(x) (any vinfo:sa vi)) (let* ((leading (filter (lambda (x) (and (pair? x) - (or (and (eq? (car x) 'method) - (length> x 2)) - (eq? (car x) '=)))) + (let ((cx (car x))) + (or (and (eq? cx 'method) (length> x 2)) + (eq? cx '=) + (eq? cx 'call))))) (take-statements-while (lambda (e) (or (atom? e) (memq (car e) '(quote top core line inert local unnecessary meta inbounds boundscheck simdloop implicit-global global globalref - const newvar = null method)))) + const newvar = null method call)))) (lam:body lam)))) - (unused (map cadr leading)) + (unused (map cadr (filter (lambda (x) (memq (car x) '(method =))) + leading))) (def (table))) ;; TODO: reorder leading statements to put assignments where the RHS is ;; `simple-atom?` at the top. (for-each (lambda (e) (set! unused (filter (lambda (v) (not (expr-uses-var e v))) unused)) - (if (memq (cadr e) unused) + (if (and (memq (car e) '(method =)) (memq (cadr e) unused)) (put! def (cadr e) #t))) leading) (for-each (lambda (v) From 1fb052e34e004e4251a67e7005176c2d31aeac0d Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 19 Aug 2016 14:26:07 -0400 Subject: [PATCH 087/114] replace branch in bit shift operators, helps #18135 (cherry picked from commit e02692f7720b000783f225692e9520376ce06fe5) ref #18149 --- base/int.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/int.jl b/base/int.jl index 79cfb0ddafb0b..6b244499f4b0b 100644 --- a/base/int.jl +++ b/base/int.jl @@ -182,11 +182,11 @@ trailing_ones(x::Integer) = trailing_zeros(~x) # note: this early during bootstrap, `>=` is not yet available # note: we only define Int shift counts here; the generic case is handled later >>(x::BitInteger, y::Int) = - 0 <= y ? x >> unsigned(y) : x << unsigned(-y) + select_value(0 <= y, x >> unsigned(y), x << unsigned(-y)) <<(x::BitInteger, y::Int) = - 0 <= y ? x << unsigned(y) : x >> unsigned(-y) + select_value(0 <= y, x << unsigned(y), x >> unsigned(-y)) >>>(x::BitInteger, y::Int) = - 0 <= y ? x >>> unsigned(y) : x << unsigned(-y) + select_value(0 <= y, x >>> unsigned(y), x << unsigned(-y)) ## integer conversions ## From 3239e25acbe5fc6877be4990eb94308c5dd3cf51 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sat, 20 Aug 2016 02:44:42 -0500 Subject: [PATCH 088/114] Try fixing LLVM 3.3 compilation (#18138) (cherry picked from commit 065a208521036f0c22b13ce8dd3c3f4b864b9b00) --- src/debuginfo.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 89e4e5d4a6859..6d0ef5b01660c 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -893,14 +893,6 @@ calc_gnu_debuglink_crc32(const void *buf, size_t size) return crc ^ ~0U; } -template -static inline void ignoreError(T &err) -{ -#if defined(LLVM39) && !defined(NDEBUG) - consumeError(err.takeError()); -#endif -} - #ifdef LLVM39 static Expected> #else @@ -958,6 +950,14 @@ extern "C" void jl_register_fptrs(uint64_t sysimage_base, void **fptrs, jl_lambd sysimg_fvars_n = n; } +template +static inline void ignoreError(T &err) +{ +#if defined(LLVM39) && !defined(NDEBUG) + consumeError(err.takeError()); +#endif +} + extern "C" void jl_refresh_dbg_module_list(void); bool jl_dylib_DI_for_fptr(size_t pointer, const llvm::object::ObjectFile **obj, llvm::DIContext **context, int64_t *slide, int64_t *section_slide, bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) From add64d9e4a53a59df8df93b0290bf6b2e43904cb Mon Sep 17 00:00:00 2001 From: Ranjan Anantharaman Date: Sat, 20 Aug 2016 13:33:40 +0530 Subject: [PATCH 089/114] Add test for 15913 and 15896 (#17991) * Add test for 15913 and 15896 * Change name of test (cherry picked from commit 6f2c1c4ac0f01a448bf4e1238936d79125d262d1) --- test/parse.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/parse.jl b/test/parse.jl index 3c625c192dc8d..50a6b61f437ae 100644 --- a/test/parse.jl +++ b/test/parse.jl @@ -667,3 +667,6 @@ end let str = "[1] [2]" @test_throws ParseError parse(str) end + +# issue 15896 and PR 15913 +@test_throws ErrorException eval(:(macro test15896(d; y=0) end)) From 293f63ac2dab274152f9b929236d5e092e8e5d09 Mon Sep 17 00:00:00 2001 From: Tracy Wadleigh Date: Sat, 20 Aug 2016 01:15:27 -0700 Subject: [PATCH 090/114] Fixes #18141, eliminating null in value when iterating over `ENV`. (#18144) * Eliminate trailing null from iterator over `ENV`. Fixes #18141. * Add failing test for #18141. (cherry picked from commit 03e7c798f9a0a5f3606abc0364b1a622218eb0c8) --- base/env.jl | 4 ++-- test/env.jl | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/base/env.jl b/base/env.jl index ee6ca3b1839fb..db7d8514823e8 100644 --- a/base/env.jl +++ b/base/env.jl @@ -92,7 +92,7 @@ if is_windows() function next(hash::EnvHash, block::Tuple{Ptr{UInt16},Ptr{UInt16}}) pos = block[1] blk = block[2] - len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) + 1 + len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos) buf = Array{UInt16}(len) unsafe_copy!(pointer(buf), pos, len) env = transcode(String, buf) @@ -100,7 +100,7 @@ if is_windows() if m === nothing error("malformed environment entry: $env") end - return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+len*2, blk)) + return (Pair{String,String}(m.captures[1], m.captures[2]), (pos+(len+1)*2, blk)) end else # !windows start(::EnvHash) = 0 diff --git a/test/env.jl b/test/env.jl index 23e8710398cab..030fe3cf9b590 100644 --- a/test/env.jl +++ b/test/env.jl @@ -63,3 +63,10 @@ end # Test for #10853 @test withenv(Dict{Any,Any}()...) do; true; end + +# Test for #18141 +for (k, v) in ENV + if length(v) > 0 + @test v[end] != '\0' + end +end From e576c87fdb04eb00ba20a14e7ec28b5de86775b1 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sun, 7 Aug 2016 05:19:47 -0700 Subject: [PATCH 091/114] Make rpath settings a bit more consistent probably addresses #18106 (cherry picked from commit fb0e10651de33660d8d638af72156a032af021d6) ref #18153 Delete some files from libssh2 and curl that keep changing on travis causing the cache to get re-uploaded (cherry picked from commit 71bb974cd97be8a9a72234b2258b034eff12e039) Install all 3 mbedtls dlls on windows and build tests (cherry picked from commit 6b50d352816fcc517597f0e32eadc0dfa53f0786) disable libpsl in curl build should fix clean build of master indent line continuations (cherry picked from commit d14f169de72c9de04b570a0f3f7a3d69c15e1653) disable using zlib in mbedtls, libssh2, and libcurl since it requires additional dev headers fix #18101 (cherry picked from commit f714e835f0e16ebec091cdd7a5c57f2eb884e50f) Cleanup: replace misleading STAGE#_DEPS variables with DEP_LIBS (cherry picked from commit 0d634a3e973585fed4cdd45aa50d1fd559752560) Improve handling of curl, mbedtls, libssh2 USE_SYSTEM fix #17884 (cherry picked from commit 4526b65d7bfaf75e904f77b03e33f554d7ab5d14) Set CMAKE_INSTALL_LIBDIR and CMAKE_INSTALL_BINDIR in CMAKE_COMMON, so libssh2 installs without an x86_64-linux-gnu multiarch prefix (cherry picked from commit bf60b56fd95c1991839efd1aad42452d1711736d) Disable building static mbedtls library since we don't need it Go back to just copying the mbedtls dll even in cygwin-cross, otherwise it puts .dll.a files that we don't need alongside the dlls (should probably fix this upstream) (cherry picked from commit 658dc242f8f32927a4fc5f8fe55d7fd061e1fc40) Also just copy libgit2 dll for its install target since otherwise it puts pkg-config files in LIB_INSTALL_DIR which we don't need - if mbedtls separates runtime from library destinations then we could separate BIN_INSTALL_DIR from LIB_INSTALL_DIR or convince libgit2 to use the standard GNUInstallDirs names (cherry picked from commit 8e3f1c53b5ad7ef67211e4a513b5e0441169c585) --- .travis.yml | 8 +++-- contrib/windows/msys_build.sh | 9 ++--- deps/Makefile | 62 +++++++++++++++++------------------ deps/arpack.mk | 7 ++-- deps/blas.mk | 7 ++-- deps/curl.mk | 24 ++++++++------ deps/libgit2.mk | 2 +- deps/libssh2.mk | 9 +++-- deps/mbedtls.mk | 10 +++--- deps/openspecfun.mk | 2 +- deps/pcre.mk | 4 +-- deps/suitesparse.mk | 4 +-- 12 files changed, 73 insertions(+), 75 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94edf04dbd460..1f7ff08e6711f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ before_install: brew install -v staticfloat/juliadeps/libgfortran; brew install -v --only-dependencies --HEAD julia; BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm37-julia)/bin/llvm-config-3.7.1 LLVM_SIZE=$(brew --prefix llvm37-julia)/bin/llvm-size-3.7.1"; - BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1 STAGE2_DEPS=utf8proc"; + BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1"; BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas"; for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND; do export BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1"; @@ -111,6 +111,10 @@ script: - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test && /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN && /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg - - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia && rm -rf julia/deps/build/julia-env + - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia && + rm -rf julia/deps/build/julia-env && + rm -rf julia/deps/build/libssh2-*/CMakeFiles/Makefile2 && + rm -rf julia/deps/build/curl-*/config.log && + rm -rf julia/deps/build/curl-*/libtool # uncomment the following if failures are suspected to be due to the out-of-memory killer # - dmesg diff --git a/contrib/windows/msys_build.sh b/contrib/windows/msys_build.sh index ae343ae1dc85b..7ada885dbf90e 100755 --- a/contrib/windows/msys_build.sh +++ b/contrib/windows/msys_build.sh @@ -178,10 +178,7 @@ echo 'override LIBLAPACKNAME = $(LIBBLASNAME)' >> Make.user # libuv since its static lib is no longer included in the binaries # openlibm since we need it as a static library to work properly # utf8proc since its headers are not in the binary download -echo 'override STAGE1_DEPS = libuv' >> Make.user -echo 'override STAGE2_DEPS = utf8proc' >> Make.user -echo 'override STAGE3_DEPS = ' >> Make.user -echo 'override STAGE4_DEPS = ' >> Make.user +echo 'override DEP_LIBS = libuv utf8proc' >> Make.user if [ -n "$USEMSVC" ]; then # Openlibm doesn't build well with MSVC right now @@ -193,14 +190,14 @@ if [ -n "$USEMSVC" ]; then make -C deps install-libuv install-utf8proc cp usr/lib/uv.lib usr/lib/libuv.a echo 'override CC += -TP' >> Make.user - echo 'override STAGE1_DEPS += dsfmt' >> Make.user + echo 'override DEP_LIBS += dsfmt' >> Make.user # Create a modified version of compile for wrapping link sed -e 's/-link//' -e 's/cl/link/g' -e 's/ -Fe/ -OUT:/' \ -e 's|$dir/$lib|$dir/lib$lib|g' deps/srccache/libuv/compile > linkld chmod +x linkld else - echo 'override STAGE1_DEPS += openlibm' >> Make.user + echo 'override DEP_LIBS += openlibm' >> Make.user make check-whitespace make VERBOSE=1 -C base version_git.jl.phony echo 'NO_GIT = 1' >> Make.user diff --git a/deps/Makefile b/deps/Makefile index 1b799d922aaee..5e59c43fb7e15 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -46,7 +46,9 @@ CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(DEPS_CFLAGS)" CXX="$(CXX) $(DEPS_CXX CMAKE_CC_ARG := $(CC_ARG) $(DEPS_CFLAGS) CMAKE_CXX_ARG := $(CXX_ARG) $(DEPS_CXXFLAGS) -CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) -DLIB_INSTALL_DIR=$(build_shlibdir) +CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) +CMAKE_COMMON += -DCMAKE_INSTALL_LIBDIR=$(build_libdir) -DCMAKE_INSTALL_BINDIR=$(build_bindir) +CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir) ifneq ($(VERBOSE), 0) CMAKE_COMMON += -DCMAKE_VERBOSE_MAKEFILE=ON endif @@ -88,31 +90,29 @@ MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_depsbindir) libd # prevent installing libs into usr/lib64 on opensuse unexport CONFIG_SITE -STAGE1_DEPS := -STAGE2_DEPS := ifeq ($(USE_GPL_LIBS), 1) -STAGE3_DEPS := suitesparse-wrapper +DEP_LIBS := suitesparse-wrapper else -STAGE3_DEPS := +DEP_LIBS := endif ifeq ($(USE_SYSTEM_LIBUV), 0) -STAGE1_DEPS += libuv +DEP_LIBS += libuv endif ifeq ($(USE_SYSTEM_LIBUNWIND), 0) ifeq ($(OS), Linux) -STAGE1_DEPS += unwind +DEP_LIBS += unwind else ifeq ($(OS), FreeBSD) -STAGE1_DEPS += unwind +DEP_LIBS += unwind else ifeq ($(OS), Darwin) -STAGE1_DEPS += osxunwind +DEP_LIBS += osxunwind endif endif ifeq ($(OS), Linux) ifeq ($(USE_SYSTEM_PATCHELF), 0) -STAGE1_DEPS += patchelf +DEP_LIBS += patchelf PATCHELF:=$(build_depsbindir)/patchelf else PATCHELF:=patchelf @@ -124,79 +124,79 @@ PATCHELF_BIN := $(CUSTOM_LD_LIBRARY_PATH) $(PATCHELF) ifeq ($(USE_SYSTEM_OPENLIBM), 0) ifeq ($(USE_SYSTEM_LIBM), 0) -STAGE1_DEPS += openlibm +DEP_LIBS += openlibm endif endif ifeq ($(USE_SYSTEM_OPENSPECFUN), 0) -STAGE1_DEPS += openspecfun +DEP_LIBS += openspecfun endif ifeq ($(USE_SYSTEM_DSFMT), 0) -STAGE1_DEPS += dsfmt +DEP_LIBS += dsfmt endif ifeq ($(USE_SYSTEM_LLVM), 0) -STAGE1_DEPS += llvm +DEP_LIBS += llvm endif ifeq ($(USE_SYSTEM_PCRE), 0) -STAGE1_DEPS += pcre +DEP_LIBS += pcre endif ifeq ($(USE_SYSTEM_BLAS), 0) -STAGE1_DEPS += openblas +DEP_LIBS += openblas ifeq ($(USE_BLAS64), 1) ifeq ($(OS), Darwin) -STAGE1_DEPS += objconv +DEP_LIBS += objconv endif endif endif ifeq ($(USE_GPL_LIBS), 1) ifeq ($(USE_SYSTEM_FFTW), 0) -STAGE1_DEPS += fftw +DEP_LIBS += fftw endif endif ifeq ($(USE_SYSTEM_GMP), 0) -STAGE1_DEPS += gmp +DEP_LIBS += gmp endif +ifeq ($(USE_SYSTEM_LIBGIT2), 0) ifeq ($(USE_SYSTEM_MBEDTLS), 0) -STAGE1_DEPS += mbedtls +DEP_LIBS += mbedtls endif ifeq ($(USE_SYSTEM_LIBSSH2), 0) -STAGE2_DEPS += libssh2 +DEP_LIBS += libssh2 endif ifneq ($(OS), WINNT) ifeq ($(USE_SYSTEM_CURL), 0) -STAGE3_DEPS += curl +DEP_LIBS += curl endif endif -ifeq ($(USE_SYSTEM_LIBGIT2), 0) -STAGE4_DEPS += libgit2 -endif +DEP_LIBS += libgit2 +endif # USE_SYSTEM_LIBGIT2 ifeq ($(USE_SYSTEM_MPFR), 0) -STAGE2_DEPS += mpfr +DEP_LIBS += mpfr endif ifeq ($(USE_SYSTEM_ARPACK), 0) -STAGE2_DEPS += arpack +DEP_LIBS += arpack endif ifeq ($(USE_GPL_LIBS), 1) ifeq ($(USE_SYSTEM_SUITESPARSE), 0) -STAGE2_DEPS += suitesparse +DEP_LIBS += suitesparse endif endif ifeq ($(USE_SYSTEM_UTF8PROC), 0) -STAGE2_DEPS += utf8proc +DEP_LIBS += utf8proc endif # Only compile standalone LAPACK if we are not using OpenBLAS. @@ -205,7 +205,7 @@ endif # build LAPACK as the vendor LAPACK may be too old (eg. Apple vecLib) ifeq ($(USE_SYSTEM_BLAS), 1) ifeq ($(USE_SYSTEM_LAPACK), 0) -STAGE2_DEPS += lapack +DEP_LIBS += lapack endif endif @@ -217,8 +217,6 @@ endif ## Common build target prefixes -DEP_LIBS := $(STAGE1_DEPS) $(STAGE2_DEPS) $(STAGE3_DEPS) $(STAGE4_DEPS) - default: install | $(build_prefix) get: $(addprefix get-, $(DEP_LIBS)) configure: $(addprefix configure-, $(DEP_LIBS)) diff --git a/deps/arpack.mk b/deps/arpack.mk index 91a5c72f05b90..7d65c10af7860 100644 --- a/deps/arpack.mk +++ b/deps/arpack.mk @@ -32,10 +32,9 @@ ARPACK_OBJ_TARGET := $(build_shlibdir)/libarpack.$(SHLIB_EXT) ARPACK_MFLAGS := F77="$(FC)" MPIF77="$(FC)" ARPACK_FFLAGS += $(FFLAGS) $(JFFLAGS) -ARPACK_FLAGS := --with-blas="$(LIBBLAS)" --with-lapack="$(LIBLAPACK)" --disable-mpi --enable-shared FFLAGS="$(ARPACK_FFLAGS)" CFLAGS="$(CFLAGS) $(ARPACK_CFLAGS)" -ifneq ($(OS),WINNT) -ARPACK_FLAGS += LDFLAGS="$(LDFLAGS) -Wl,-rpath,'$(build_libdir)'" -endif +ARPACK_FLAGS := --with-blas="$(LIBBLAS)" --with-lapack="$(LIBLAPACK)" \ + --disable-mpi --enable-shared FFLAGS="$(ARPACK_FFLAGS)" \ + CFLAGS="$(CFLAGS) $(ARPACK_CFLAGS)" LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)" # ARPACK-NG upstream keeps changing their download filenames $(SRCDIR)/srccache/arpack-ng-$(ARPACK_VER).tar.gz: | $(SRCDIR)/srccache diff --git a/deps/blas.mk b/deps/blas.mk index fcd6cd8074bb0..809c4c7297932 100644 --- a/deps/blas.mk +++ b/deps/blas.mk @@ -200,10 +200,9 @@ LAPACK_OBJ_TARGET := LAPACK_OBJ_SOURCE := endif -LAPACK_MFLAGS := NOOPT="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS) -O0" OPTS="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS)" FORTRAN="$(FC)" LOADER="$(FC)" -ifneq ($(OS),WINNT) -LAPACK_MFLAGS += BLASLIB="-Wl,-rpath,'$(build_libdir)' $(LIBBLAS)" -endif +LAPACK_MFLAGS := NOOPT="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS) -O0" \ + OPTS="$(FFLAGS) $(JFFLAGS) $(GFORTBLAS_FFLAGS)" FORTRAN="$(FC)" \ + LOADER="$(FC)" BLASLIB="$(RPATH_ESCAPED_ORIGIN) $(LIBBLAS)" $(SRCDIR)/srccache/lapack-$(LAPACK_VER).tgz: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ http://www.netlib.org/lapack/$(notdir $@) diff --git a/deps/curl.mk b/deps/curl.mk index 060b41b7969a1..893e15aa859ef 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -3,29 +3,33 @@ CURL_SRC_TARGET := $(BUILDDIR)/curl-$(CURL_VER)/lib/.libs/libcurl.$(SHLIB_EXT) CURL_OBJ_TARGET := $(build_shlibdir)/libcurl.$(SHLIB_EXT) -ifneq ($(OS),WINNT) CURL_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) -endif $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://curl.haxx.se/download/curl-$(CURL_VER).tar.bz2 -$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(MBEDTLS_OBJ_TARGET) $(LIBSSH2_OBJ_TARGET) +$(SRCDIR)/srccache/curl-$(CURL_VER)/configure: $(SRCDIR)/srccache/curl-$(CURL_VER).tar.bz2 $(JLCHECKSUM) $< cd $(dir $<) && $(TAR) jxf $(notdir $<) touch -c $@ +ifeq ($(USE_SYSTEM_MBEDTLS), 0) +$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(MBEDTLS_OBJ_TARGET) +endif +ifeq ($(USE_SYSTEM_LIBSSH2), 0) +$(BUILDDIR)/curl-$(CURL_VER)/config.status: $(LIBSSH2_OBJ_TARGET) +endif $(BUILDDIR)/curl-$(CURL_VER)/config.status: $(SRCDIR)/srccache/curl-$(CURL_VER)/configure mkdir -p $(dir $@) cd $(dir $@) && \ $< $(CONFIGURE_COMMON) --includedir=$(build_includedir) \ - --without-ssl --without-gnutls --without-gssapi \ - --without-libidn --without-libmetalink --without-librtmp \ - --without-nghttp2 --without-nss --without-polarssl \ - --without-spnego --disable-ares --disable-ldap \ - --disable-ldaps --without-zsh-functions-dir \ - --with-libssh2=$(build_prefix) --with-mbedtls=$(build_prefix) \ - CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" + --without-ssl --without-gnutls --without-gssapi --without-zlib \ + --without-libidn --without-libmetalink --without-librtmp \ + --without-nghttp2 --without-nss --without-polarssl \ + --without-spnego --without-libpsl --disable-ares \ + --disable-ldap --disable-ldaps --without-zsh-functions-dir \ + --with-libssh2=$(build_prefix) --with-mbedtls=$(build_prefix) \ + CFLAGS="$(CFLAGS) $(CURL_CFLAGS)" LDFLAGS="$(LDFLAGS) $(CURL_LDFLAGS)" touch -c $@ $(CURL_SRC_TARGET): $(BUILDDIR)/curl-$(CURL_VER)/config.status diff --git a/deps/libgit2.mk b/deps/libgit2.mk index f17bea8ff3345..c1e0a2cc82b6b 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -71,7 +71,7 @@ endif echo 1 > $@ $(LIBGIT2_OBJ_TARGET): $(LIBGIT2_OBJ_SOURCE) | $(build_shlibdir) -ifeq ($(BUILD_OS),WINNT) +ifeq ($(OS),WINNT) cp $< $@ else $(call make-install,$(LIBGIT2_SRC_DIR),) diff --git a/deps/libssh2.mk b/deps/libssh2.mk index eb2282a882e12..48f111dfcc5e9 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -8,7 +8,7 @@ LIBSSH2_OBJ_SOURCE := $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/src/libssh2.$(SHLIB_EXT) LIBSSH2_OBJ_TARGET := $(build_shlibdir)/libssh2.$(SHLIB_EXT) LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \ - -DCMAKE_BUILD_TYPE=Release + -DCMAKE_BUILD_TYPE=Release ifeq ($(OS),WINNT) LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF @@ -16,7 +16,7 @@ ifeq ($(BUILD_OS),WINNT) LIBSSH2_OPTS += -G"MSYS Makefiles" endif else -LIBSSH2_OPTS += -DCRYPTO_BACKEND=mbedTLS -DENABLE_ZLIB_COMPRESSION=ON +LIBSSH2_OPTS += -DCRYPTO_BACKEND=mbedTLS -DENABLE_ZLIB_COMPRESSION=OFF endif ifeq ($(OS),Linux) @@ -30,7 +30,10 @@ $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-encryptedpem.patch-applied: | $(SR cd $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR) && patch -p1 -f < $(SRCDIR)/patches/libssh2-encryptedpem.patch echo 1 > $@ -$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-encryptedpem.patch-applied $(MBEDTLS_OBJ_TARGET) +ifeq ($(USE_SYSTEM_MBEDTLS), 0) +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(MBEDTLS_OBJ_TARGET) +endif +$(BUILDDIR)/$(LIBSSH2_SRC_DIR)/Makefile: $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/CMakeLists.txt $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-mbedtls.patch-applied $(SRCDIR)/srccache/$(LIBSSH2_SRC_DIR)/libssh2-encryptedpem.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LIBSSH2_OPTS) diff --git a/deps/mbedtls.mk b/deps/mbedtls.mk index 150a39f2a1196..f9a340f83c9c1 100644 --- a/deps/mbedtls.mk +++ b/deps/mbedtls.mk @@ -11,15 +11,11 @@ MBEDTLS_OBJ_SOURCE := $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedcrypto.$ MBEDTLS_OBJ_TARGET := $(build_shlibdir)/libmbedcrypto.$(SHLIB_EXT) MBEDTLS_OPTS := $(CMAKE_COMMON) -DUSE_SHARED_MBEDTLS_LIBRARY=ON \ - -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release + -DUSE_STATIC_MBEDTLS_LIBRARY=OFF -DENABLE_PROGRAMS=OFF -DCMAKE_BUILD_TYPE=Release -ifeq ($(OS),WINNT) MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=OFF ifeq ($(BUILD_OS),WINNT) -MBEDTLS_OPTS += -G"MSYS Makefiles" -DENABLE_TESTING=OFF -endif -else -MBEDTLS_OPTS += -DENABLE_ZLIB_SUPPORT=ON +MBEDTLS_OPTS += -G"MSYS Makefiles" endif ifeq ($(OS),Linux) @@ -58,6 +54,8 @@ endif $(MBEDTLS_OBJ_TARGET): $(MBEDTLS_OBJ_SOURCE) | $(build_shlibdir) ifeq ($(OS), WINNT) cp $^ $(build_shlibdir) + cp $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedx509.$(SHLIB_EXT) $(build_shlibdir) + cp $(BUILDDIR)/mbedtls-$(MBEDTLS_VER)/library/libmbedtls.$(SHLIB_EXT) $(build_shlibdir) else $(call make-install,mbedtls-$(MBEDTLS_VER),) endif diff --git a/deps/openspecfun.mk b/deps/openspecfun.mk index 6e3ccce12d02a..f6c5dfead968f 100644 --- a/deps/openspecfun.mk +++ b/deps/openspecfun.mk @@ -13,7 +13,7 @@ OPENSPECFUN_OBJ_TARGET := $(build_shlibdir)/libopenspecfun.$(SHLIB_EXT) OPENSPECFUN_OBJ_SOURCE := $(BUILDDIR)/$(OPENSPECFUN_SRC_DIR)/libopenspecfun.$(SHLIB_EXT) OPENSPECFUN_FLAGS := ARCH="$(ARCH)" CC="$(CC)" FC="$(FC)" AR="$(AR)" OS="$(OS)" \ USECLANG=$(USECLANG) USEGCC=$(USEGCC) FFLAGS="$(JFFLAGS)" \ - CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" LDFLAGS="$(RPATH_ESCAPED_ORIGIN)" + CFLAGS="$(CFLAGS) $(OPENSPECFUN_CFLAGS)" LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN)" ifeq ($(USE_SYSTEM_LIBM),0) OPENSPECFUN_FLAGS += USE_OPENLIBM=1 diff --git a/deps/pcre.mk b/deps/pcre.mk index 5ae612452e763..b251f0c8aeae1 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -5,9 +5,7 @@ PCRE_OBJ_TARGET := $(build_shlibdir)/libpcre2-8.$(SHLIB_EXT) # Force optimization for PCRE flags (Issue #11668) PCRE_CFLAGS := -O3 -ifneq ($(OS),WINNT) -PCRE_LDFLAGS := "-Wl,-rpath,'$(build_libdir)'" -endif +PCRE_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) $(SRCDIR)/srccache/pcre2-$(PCRE_VER).tar.bz2: | $(SRCDIR)/srccache $(JLDOWNLOAD) $@ https://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre2-$(PCRE_VER).tar.bz2 diff --git a/deps/suitesparse.mk b/deps/suitesparse.mk index 975572e762466..87a07adfe898d 100644 --- a/deps/suitesparse.mk +++ b/deps/suitesparse.mk @@ -22,9 +22,7 @@ ifneq ($(OS), WINNT) SUITE_SPARSE_LIB += -lrt endif endif -ifneq ($(OS), WINNT) -SUITE_SPARSE_LIB += -Wl,-rpath,'$(build_libdir)' -endif +SUITE_SPARSE_LIB += $(RPATH_ESCAPED_ORIGIN) SUITESPARSE_MFLAGS := CC="$(CC)" CXX="$(CXX)" F77="$(FC)" AR="$(AR)" RANLIB="$(RANLIB)" BLAS="$(LIBBLAS)" LAPACK="$(LIBLAPACK)" \ INSTALL_LIB="$(build_libdir)" INSTALL_INCLUDE="$(build_includedir)" LIB="$(SUITE_SPARSE_LIB)" \ UMFPACK_CONFIG="$(UMFPACK_CONFIG)" CHOLMOD_CONFIG="$(CHOLMOD_CONFIG)" SPQR_CONFIG="$(SPQR_CONFIG)" From d0a632b50be3d20cf863340f6c1c7ab0ae6d41f1 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 20 Aug 2016 01:38:32 -0700 Subject: [PATCH 092/114] circcopy! needs qualification with Base. for 0.5 where it is not exported, ref #17919 --- test/offsetarray.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/offsetarray.jl b/test/offsetarray.jl index c2ba9687c4ccc..86a2d5b5c91cc 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -341,7 +341,7 @@ v = OffsetArray(rand(8), (-2,)) src = reshape(collect(1:16), (4,4)) dest = OffsetArray(Array{Int}(4,4), (-1,1)) -circcopy!(dest, src) +Base.circcopy!(dest, src) @test parent(dest) == [8 12 16 4; 5 9 13 1; 6 10 14 2; 7 11 15 3] @test dest[1:3,2:4] == src[1:3,2:4] @@ -359,8 +359,8 @@ for s = -5:5 thisc = c[mod1(i+s+5,5)] @test_approx_eq fft(thisa) thisc @test_approx_eq fft(thisa, 1) thisc - @test_approx_eq ifft(fft(thisa)) circcopy!(a1, thisa) - @test_approx_eq ifft(fft(thisa, 1), 1) circcopy!(a1, thisa) + @test_approx_eq ifft(fft(thisa)) Base.circcopy!(a1, thisa) + @test_approx_eq ifft(fft(thisa, 1), 1) Base.circcopy!(a1, thisa) @test_approx_eq rfft(thisa) thisc[1:3] @test_approx_eq rfft(thisa, 1) thisc[1:3] @test_approx_eq irfft(rfft(thisa, 1), 5, 1) a1 From 6da641f35796f51078085859064052e4c0f05262 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Sat, 20 Aug 2016 05:14:18 -0400 Subject: [PATCH 093/114] Fix doctests --- doc/manual/conversion-and-promotion.rst | 4 ++-- doc/manual/mathematical-operations.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/conversion-and-promotion.rst b/doc/manual/conversion-and-promotion.rst index 11651893358c3..f9cfd8fe3950f 100644 --- a/doc/manual/conversion-and-promotion.rst +++ b/doc/manual/conversion-and-promotion.rst @@ -91,12 +91,12 @@ action: Float64 julia> a = Any[1 2 3; 4 5 6] - 2x3 Array{Any,2}: + 2×3 Array{Any,2}: 1 2 3 4 5 6 julia> convert(Array{Float64}, a) - 2x3 Array{Float64,2}: + 2×3 Array{Float64,2}: 1.0 2.0 3.0 4.0 5.0 6.0 diff --git a/doc/manual/mathematical-operations.rst b/doc/manual/mathematical-operations.rst index c7925dea544e6..36fba25e8c46f 100644 --- a/doc/manual/mathematical-operations.rst +++ b/doc/manual/mathematical-operations.rst @@ -427,7 +427,7 @@ The following examples show the different forms. julia> round(Int8,127.6) ERROR: InexactError() - in trunc(::Type{Int8}, ::Float64) at ./float.jl:456 + in trunc(::Type{Int8}, ::Float64) at ./float.jl:458 in round(::Type{Int8}, ::Float64) at ./float.jl:211 ... From f5ed7678ffa3abfd835ec6513aab1fbfd534d7d3 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 13 Aug 2016 08:03:55 -0500 Subject: [PATCH 094/114] Force inlining on indices(A, d) (cherry picked from commit 95b858a65f50804b992e0f1979057e63e587822b) ref #18014 --- base/abstractarray.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 045d255ba91b3..5a2f0c53ff61a 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -52,7 +52,11 @@ size{N}(x, d1::Integer, d2::Integer, dx::Vararg{Integer, N}) = (size(x, d1), siz Returns the valid range of indices for array `A` along dimension `d`. """ -indices{T,N}(A::AbstractArray{T,N}, d) = d <= N ? indices(A)[d] : OneTo(1) +function indices{T,N}(A::AbstractArray{T,N}, d) + @_inline_meta + d <= N ? indices(A)[d] : OneTo(1) +end + """ indices(A) From bfa4e6200f5ba019984aa13deeb3c518a7e21d2d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 16 Aug 2016 11:07:04 -0400 Subject: [PATCH 095/114] Clear the IRBuilder's insertion point after emitting a function. The function might get finalized, invalidating the IP. However, in some cases this invalid IP may get saved and restored, accessing the invalid IP while doing so. Example code path accessing an invalid IP: -> jl_cfunction_object (nested_compile=true, but doesn't change IP) -> gen_cfun_wrapper -> jl_compile_linfo (saves and restores invalid IP) (cherry picked from commit ebd24a84f9f3746b78d4c9ae09f9a8e2bf2db793, ref #18054) --- src/codegen.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index e7d07688e9dad..db21394b3e61c 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4746,6 +4746,8 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func } } + builder.ClearInsertionPoint(); + // step 14, Apply LLVM level inlining for(std::vector::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) { Function *inlinef = (*it)->getCalledFunction(); From fa24dff5812644d0f2acdd72e9899866f915359b Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 16 Aug 2016 20:27:57 +0200 Subject: [PATCH 096/114] Add test for #18054. Note that this test requires a memory sanitizer (ASAN, valgrind) to detect failure. (cherry picked from commit a5fdd74abcf53e5858b98833d64fa6bf84954607, ref #18054) --- test/core.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/core.jl b/test/core.jl index 0c39246b57bab..66ba2ef72368f 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4470,6 +4470,13 @@ let @test k(1) == 1 end +# PR #18054: compilation of cfunction leaves IRBuilder in bad state, +# causing heap-use-after-free when compiling f18054 +function f18054() + return Cint(0) +end +cfunction(f18054, Cint, ()) + # issue #18085 f18085(a,x...) = (0,) for (f,g) in ((:asin,:sin), (:acos,:cos)) From 51d38f57aa68c9c35a520dbcaf570eb15ca6934e Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 19 Aug 2016 09:53:20 -0400 Subject: [PATCH 097/114] Expose MEMDEBUG, and use it to fix the #14173 test (cherry picked from commit 989e544445d38884e40403f20d1fced51b1141f6, ref #18143) --- src/intrinsics.cpp | 8 ++++++++ test/misc.jl | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 444a76b853f7d..1d9844e20e944 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -50,6 +50,14 @@ JL_DLLEXPORT uint32_t jl_get_LLVM_VERSION(void) ; } +extern "C" JL_DLLEXPORT int8_t jl_is_memdebug() { +#ifdef MEMDEBUG + return true; +#else + return false; +#endif +} + /* low-level intrinsics design: TODO: fix description below functions like add_int expect unboxed values of matching bit-length. diff --git a/test/misc.jl b/test/misc.jl index bf3795ffb8051..e91c3ca135b8c 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -230,7 +230,8 @@ module Tmp14173 A = randn(2000, 2000) end whos(IOBuffer(), Tmp14173) # warm up -@test @allocated(whos(IOBuffer(), Tmp14173)) < 10000 +const MEMDEBUG = ccall(:jl_is_memdebug, Bool, ()) +@test @allocated(whos(IOBuffer(), Tmp14173)) < (MEMDEBUG ? 20000 : 8000) ## test conversion from UTF-8 to UTF-16 (for Windows APIs) From be350f2bf8b0329a2ccacfaf0db6b8495546c6f7 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 20 Aug 2016 14:09:12 -0400 Subject: [PATCH 098/114] Increase the signal stack size. This makes the SIGINT signal handler work in combination with ASAN. (partially cherry picked from commit 03c3c7004bec19eadb5b5dee70183d63ca765eec, ref #17727) --- src/julia_internal.h | 3 --- src/signals-unix.c | 12 ++++-------- src/threading.c | 7 ------- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/julia_internal.h b/src/julia_internal.h index 8346c247e5808..6eec5c076d43a 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -49,9 +49,6 @@ void jl_call_tracer(tracer_cb callback, jl_value_t *tracee); extern size_t jl_page_size; extern jl_function_t *jl_typeinf_func; -#if defined(JL_USE_INTEL_JITEVENTS) -extern unsigned sig_stack_size; -#endif JL_DLLEXPORT extern int jl_lineno; JL_DLLEXPORT extern const char *jl_filename; diff --git a/src/signals-unix.c b/src/signals-unix.c index d0814741fe9c5..23843d3e9b5b9 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -28,14 +28,10 @@ #define HAVE_TIMER #endif -#if defined(JL_USE_INTEL_JITEVENTS) -unsigned sig_stack_size = SIGSTKSZ; -#elif defined(_CPU_AARCH64_) -// The default SIGSTKSZ causes stack overflow in libunwind. -#define sig_stack_size (1 << 16) -#else -#define sig_stack_size SIGSTKSZ -#endif +// 8M signal stack, same as default stack size and enough +// for reasonable finalizers. +// Should also be enough for parallel GC when we have it =) +#define sig_stack_size (8 * 1024 * 1024) static bt_context_t *jl_to_bt_context(void *sigctx) { diff --git a/src/threading.c b/src/threading.c index 6b1d373c7f1f6..7ff20c565338a 100644 --- a/src/threading.c +++ b/src/threading.c @@ -778,13 +778,6 @@ void jl_init_threading(void) jl_all_tls_states = &_jl_all_tls_states; jl_n_threads = 1; -#if defined(__linux__) && defined(JL_USE_INTEL_JITEVENTS) - if (jl_using_intel_jitevents) - // Intel VTune Amplifier needs at least 64k for alternate stack. - if (SIGSTKSZ < 1<<16) - sig_stack_size = 1<<16; -#endif - ti_init_master_thread(); } From 727daa6e0ad2d52e83b8da1447068187e193e689 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 16 Aug 2016 19:58:37 +0200 Subject: [PATCH 099/114] ASAN: provide default options. (cherry picked from commit 2d9f3a78adedc1b63a3fcba2f2d238a6245aeeae, ref #18067) --- src/init.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/init.c b/src/init.c index a2d74fe60cc91..d4bf640a181fe 100644 --- a/src/init.c +++ b/src/init.c @@ -46,6 +46,14 @@ extern BOOL (WINAPI *hSymRefreshModuleList)(HANDLE); #include #endif +#ifdef JL_ASAN_ENABLED +JL_DLLEXPORT const char* __asan_default_options() { + return "allow_user_segv_handler=1:detect_leaks=0"; + // FIXME: enable LSAN after fixing leaks & defining __lsan_default_suppressions(), + // or defining __lsan_default_options = exitcode=0 once publicly available +} +#endif + static const char system_image_path[256] = "\0" JL_SYSTEM_IMAGE_PATH; jl_options_t jl_options = { 0, // quiet @@ -628,17 +636,6 @@ void _julia_init(JL_IMAGE_SEARCH rel) } #endif - -#ifdef JL_ASAN_ENABLED - const char *asan_options = getenv("ASAN_OPTIONS"); - if (!asan_options || !(strstr(asan_options, "allow_user_segv_handler=1") || - strstr(asan_options, "handle_segv=0"))) { - jl_printf(JL_STDERR,"WARNING: ASAN overrides Julia's SIGSEGV handler; " - "disable SIGSEGV handling or allow custom handlers.\n"); - } - -#endif - jl_init_threading(); jl_gc_init(); From f3566be11da0cb16d8b8a2cadb554dc8445896aa Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 16 Aug 2016 20:19:27 +0200 Subject: [PATCH 100/114] ASAN/MSAN: document use of sanitizers. (cherry picked from commit 030ff40d8f43fa4f98a44e81bc83e9c8ed2370ef, ref #18067) --- doc/devdocs/C.rst | 1 + doc/devdocs/sanitizers.rst | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 doc/devdocs/sanitizers.rst diff --git a/doc/devdocs/C.rst b/doc/devdocs/C.rst index 60150eb777d9b..b5866a7fa18ea 100644 --- a/doc/devdocs/C.rst +++ b/doc/devdocs/C.rst @@ -12,3 +12,4 @@ backtraces debuggingtips valgrind + sanitizers diff --git a/doc/devdocs/sanitizers.rst b/doc/devdocs/sanitizers.rst new file mode 100644 index 0000000000000..262e259825d5c --- /dev/null +++ b/doc/devdocs/sanitizers.rst @@ -0,0 +1,44 @@ +***************** +Sanitizer support +***************** + +General considerations +---------------------- + +Using Clang's sanitizers obviously require you to use Clang (``USECLANG=1``), but there's +another catch: most sanitizers require a run-time library, provided by the host compiler, +while the instrumented code generated by Julia's JIT relies on functionality from that +library. This implies that the LLVM version of your host compiler matches that of the LLVM +library used within Julia. + +An easy solution is to have an dedicated build folder for providing a matching toolchain, by +building with ``BUILD_LLVM_CLANG=1`` and overriding ``LLVM_USE_CMAKE=1`` (Autotool-based +builds are incompatible with ASAN). You can then refer to this toolchain from another build +folder by specifying ``USECLANG=1`` while overriding the ``CC`` and ``CXX`` variables. + + +Address Sanitizer (ASAN) +------------------------ + +For detecting or debugging memory bugs, you can use Clang's `address sanitizer (ASAN) +`_. By compiling with +``SANITIZE=1`` you enable ASAN for the Julia compiler and its generated code. In addition, +you can specify ``LLVM_SANITIZE=1`` to sanitize the LLVM library as well. Note that these +options incur a high performance and memory cost. For example, using ASAN for Julia and LLVM +makes ``testall1`` takes 8-10 times as long while using 20 times as much memory (this can +be reduced to respectively a factor of 3 and 4 by using the options described below). + +By default, Julia sets the ``allow_user_segv_handler=1`` ASAN flag, which is required for +signal delivery to work properly. You can define other options using the ``ASAN_OPTIONS`` +environment flag, in which case you'll need to repeat the default option mentioned before. +For example, memory usage can be reduced by specifying ``fast_unwind_on_malloc=0`` and +``malloc_context_size=2``, at the cost of backtrace accuracy. For now, Julia also sets +``detect_leaks=0``, but this should be removed in the future. + + +Memory Sanitizer (MSAN) +----------------------- + +For detecting use of uninitialized memory, you can use Clang's `memory sanitizer (MSAN) +`_ by compiling with +``SANITIZE_MEMORY=1``. From 9e1dbfd0dc2c6014b4a7f5872ffe7423b63a65b2 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 10 Aug 2016 15:45:54 -0400 Subject: [PATCH 101/114] remove some unnecessary anonymous functions Transform `x->f(x)` to `f`. (cherry picked from commit 9cead4f7874a2aa9b1a8d6adc59d836683c45e61) ref #17974 --- base/abstractarray.jl | 2 +- base/linalg/arpack.jl | 6 +++--- base/multidimensional.jl | 2 +- base/pkg/types.jl | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 5a2f0c53ff61a..514344215b12c 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -64,7 +64,7 @@ Returns the tuple of valid indices for array `A`. """ function indices(A) @_inline_meta - map(s->OneTo(s), size(A)) + map(OneTo, size(A)) end # Performance optimization: get rid of a branch on `d` in `indices(A, diff --git a/base/linalg/arpack.jl b/base/linalg/arpack.jl index 9b27aa3bc019b..26a0c55e1dbb6 100644 --- a/base/linalg/arpack.jl +++ b/base/linalg/arpack.jl @@ -114,15 +114,15 @@ function eupd_wrapper(T, n::Integer, sym::Bool, cmplx::Bool, bmat::String, select = Array{BlasInt}(ncv) info = zeros(BlasInt, 1) - dmap = x->abs(x) + dmap = abs if iparam[7] == 3 # shift-and-invert dmap = x->abs(1./(x-sigma)) elseif which == "LR" || which == "LA" || which == "BE" - dmap = x->real(x) + dmap = real elseif which == "SR" || which == "SA" dmap = x->-real(x) elseif which == "LI" - dmap = x->imag(x) + dmap = imag elseif which == "SI" dmap = x->-imag(x) end diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 660773dbe1790..0a2b9489cec51 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -79,7 +79,7 @@ end CartesianRange{N}(index::CartesianIndex{N}) = CartesianRange(one(index), index) CartesianRange(::Tuple{}) = CartesianRange{CartesianIndex{0}}(CartesianIndex{0}(()),CartesianIndex{0}(())) CartesianRange{N}(sz::NTuple{N,Int}) = CartesianRange(CartesianIndex(sz)) -CartesianRange{N}(rngs::NTuple{N,Union{Integer,AbstractUnitRange}}) = CartesianRange(CartesianIndex(map(r->first(r), rngs)), CartesianIndex(map(r->last(r), rngs))) +CartesianRange{N}(rngs::NTuple{N,Union{Integer,AbstractUnitRange}}) = CartesianRange(CartesianIndex(map(first, rngs)), CartesianIndex(map(last, rngs))) ndims(R::CartesianRange) = length(R.start) ndims{I<:CartesianIndex}(::Type{CartesianRange{I}}) = length(I) diff --git a/base/pkg/types.jl b/base/pkg/types.jl index 152b3e5af0c24..9d5556a370b66 100644 --- a/base/pkg/types.jl +++ b/base/pkg/types.jl @@ -37,7 +37,7 @@ end VersionSet(versions::VersionNumber...) = VersionSet(VersionNumber[versions...]) show(io::IO, s::VersionSet) = join(io, s.intervals, " ∪ ") -isempty(s::VersionSet) = all(i->isempty(i), s.intervals) +isempty(s::VersionSet) = all(isempty, s.intervals) in(v::VersionNumber, s::VersionSet) = any(i->in(v,i), s.intervals) function intersect(A::VersionSet, B::VersionSet) ivals = vec([ intersect(a,b) for a in A.intervals, b in B.intervals ]) From 2082241a539d092f65b0ed218af911302d488fff Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 17 Aug 2016 13:18:49 +0200 Subject: [PATCH 102/114] do not catch everything in isassigned (cherry picked from commit bc0348e298f57e9774fefeb75ab848d581276c76) ref #18075 --- base/abstractarray.jl | 9 ++++++--- test/abstractarray.jl | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 514344215b12c..d7fee32340826 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -182,12 +182,15 @@ function _strides{M,T,N}(out::NTuple{M}, A::AbstractArray{T,N}) end function isassigned(a::AbstractArray, i::Int...) - # TODO try a[i...] true - catch - false + catch e + if isa(e, BoundsError) || isa(e, UndefRefError) + return false + else + rethrow(e) + end end end diff --git a/test/abstractarray.jl b/test/abstractarray.jl index ab9f68f53ef2e..db26e0099f896 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -408,6 +408,13 @@ function test_primitives{T}(::Type{T}, shape, ::Type{TestAbstractArray}) @test convert(Array, X) == X end +let + type TestThrowNoGetindex{T} <: AbstractVector{T} end + Base.length(::TestThrowNoGetindex) = 2 + Base.size(::TestThrowNoGetindex) = (2,) + @test_throws ErrorException isassigned(TestThrowNoGetindex{Float64}(), 1) +end + function test_in_bounds(::Type{TestAbstractArray}) n = rand(2:5) sz = rand(2:5, n) From 458c832428c6b79084f9b4e02ebe0905eb560b84 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 16 Aug 2016 15:24:10 -0400 Subject: [PATCH 103/114] fix #10633, remove redundant definitions for `map(Integer, a)` etc. (cherry picked from commit 90753676b686fc8d3511a9b6112f3054ec0320f7) ref #18062 --- base/abstractarray.jl | 4 ---- test/functional.jl | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index d7fee32340826..fbecc43efa4f8 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -717,10 +717,6 @@ of_indices(x, y) = similar(dims->y, oftype(indices(x), indices(y))) full(x::AbstractArray) = x -map(::Type{Integer}, a::Array) = map!(Integer, similar(a,typeof(Integer(one(eltype(a))))), a) -map(::Type{Signed}, a::Array) = map!(Signed, similar(a,typeof(Signed(one(eltype(a))))), a) -map(::Type{Unsigned}, a::Array) = map!(Unsigned, similar(a,typeof(Unsigned(one(eltype(a))))), a) - ## range conversions ## map{T<:Real}(::Type{T}, r::StepRange) = T(r.start):T(r.step):T(last(r)) diff --git a/test/functional.jl b/test/functional.jl index 5ed8f3ce51db8..4b8db1d0af750 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -32,6 +32,10 @@ end # maps of strings (character arrays) -- string.jl @test map((c)->Char(c+1), "abcDEF") == "bcdEFG" +# issue #10633 +@test isa(map(Integer, Any[1, 2]), Vector{Int}) +@test isa(map(Integer, Any[]), Vector{Integer}) + # filter -- array.jl @test isequal(filter(x->(x>1), [0 1 2 3 2 1 0]), [2, 3, 2]) # TODO: @test_throws isequal(filter(x->x+1, [0 1 2 3 2 1 0]), [2, 3, 2]) From 55a5b12f0dcac086b6fafa6d6273a0c45ffa483f Mon Sep 17 00:00:00 2001 From: ranjanan Date: Thu, 11 Aug 2016 12:11:05 +0530 Subject: [PATCH 104/114] Add test for PR #17803 The PR fixes jl_static_show for bitstypes (cherry picked from commit 718391db9ec76d4a12c128d2ae6165ad90a1fb8c) ref #17961 Change test to Int128(-1) (cherry picked from commit 462a1ebd9348ba46f7983b6e0b17a8cfcd9e3e4d) --- test/spawn.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/spawn.jl b/test/spawn.jl index 21ce279b5eb3e..5379300b83ed4 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -428,3 +428,11 @@ if is_unix() end end end + +# Test for PR 17803 +let p=Pipe() + Base.link_pipe(p; julia_only_read=true, julia_only_write=true) + ccall(:jl_static_show, Void, (Ptr{Void}, Any), p.in, Int128(-1)) + @async close(p.in) + @test readstring(p.out) == "Int128(0xffffffffffffffffffffffffffffffff)" +end From 38bec84842d52a5e5b06fc83f3b1d1249b80b165 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 18 Aug 2016 16:57:37 -0400 Subject: [PATCH 105/114] fix more of the #18017 regression (cherry picked from commit b32fcc9765ae848a5b4be0c35795e2e330d4fb8e) ref #18126 --- base/inference.jl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index e8b319573322c..79edb33b96cbd 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2261,12 +2261,15 @@ function effect_free(e::ANY, linfo::LambdaInfo, allow_volatile::Bool) if !allow_volatile if is_known_call(e, arrayref, linfo) || is_known_call(e, arraylen, linfo) return false - elseif is_known_call(e, getfield, linfo) && !isa(exprtype(e,linfo), Const) - # first argument must be immutable to ensure e is affect_free - a = ea[2] - typ = widenconst(exprtype(a, linfo)) - if !isa(typ, DataType) || typ.mutable || typ.abstract - return false + elseif is_known_call(e, getfield, linfo) + et = exprtype(e,linfo) + if !isa(et,Const) && !(isType(et) && isleaftype(et)) + # first argument must be immutable to ensure e is affect_free + a = ea[2] + typ = widenconst(exprtype(a, linfo)) + if !isa(typ, DataType) || typ.mutable || typ.abstract + return false + end end end end From 285ef3e64a8b3e122a379556ec38dd0c10944a74 Mon Sep 17 00:00:00 2001 From: Michael Hatherly Date: Sun, 21 Aug 2016 06:21:21 +0200 Subject: [PATCH 106/114] Add note about docstrings for aliases (#18157) As mentioned in https://github.com/JuliaLang/julia/pull/18041#issuecomment-241189817 docstrings attached to aliases of functions should be avoided where possible. (cherry picked from commit 3ed55a404eb425229f0556a4f7fdc614008c0ed6) --- doc/manual/documentation.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/manual/documentation.rst b/doc/manual/documentation.rst index a6d481f340a58..b40dedf5038a0 100644 --- a/doc/manual/documentation.rst +++ b/doc/manual/documentation.rst @@ -441,6 +441,32 @@ Global Variables Adds docstring ``"..."`` to the ``Binding``\ s ``a``, ``b``, and ``c``. +.. note:: + + When a ``const`` definition is only used to define an alias of another definition, such + as is the case with the function ``div`` and its alias ``÷`` in ``Base``, do not + document the alias and instead document the actual function. + + If the alias is documented and not the real definition then the docsystem (``?`` mode) + will not return the docstring attached to the alias when the real definition is + searched for. + + For example you should write + + .. code-block:: julia + + "..." + f(x) = x + 1 + const alias = f + + rather than + + .. code-block:: julia + + f(x) = x + 1 + "..." + const alias = f + .. code-block:: julia "..." From a6a7fd2c6286ba44c1dde54a7ec76210e3407f17 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Fri, 19 Aug 2016 10:32:29 +0800 Subject: [PATCH 107/114] Use unique identifier to avoid incorrect ditype reuse (cherry picked from commit 05cb1f54e84540e128a80d1e5b5ae1d876b046c0) ref #18134 --- src/cgutils.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 9ad239345595d..4a93d229b6df7 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -96,7 +96,18 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed jl_datatype_t *jdt = (jl_datatype_t*)jt; if (jdt->ditype != NULL) { #ifdef LLVM37 - return (llvm::DIType*)jdt->ditype; + DIType* t = (DIType*)jdt->ditype; +#ifndef LLVM39 + // On LLVM 3.7 and 3.8, DICompositeType with a unique name + // are ref'd by their unique name and needs to be explicitly + // retained in order to be used in the module. + if (auto *Composite = dyn_cast(t)) { + if (Composite->getRawIdentifier()) { + dbuilder->retainType(Composite); + } + } +#endif + return t; #else return DIType((llvm::MDNode*)jdt->ditype); #endif @@ -130,17 +141,22 @@ static DIType julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed else if (jl_is_structtype(jt)) { jl_datatype_t *jst = (jl_datatype_t*)jt; size_t ntypes = jl_datatype_nfields(jst); + const char *tname = jl_symbol_name(jdt->name->name); + std::stringstream unique_name; + unique_name << tname << "_" << globalUnique++; llvm::DICompositeType *ct = dbuilder->createStructType( NULL, // Scope - jl_symbol_name(jdt->name->name), // Name + tname, // Name NULL, // File 0, // LineNumber 8 * jdt->size, // SizeInBits - 8 * jdt->layout->alignment, // AlignmentInBits + 8 * jdt->layout->alignment, // AlignInBits 0, // Flags NULL, // DerivedFrom DINodeArray(), // Elements - dwarf::DW_LANG_Julia // RuntimeLanguage + dwarf::DW_LANG_Julia, // RuntimeLanguage + nullptr, // VTableHolder + unique_name.str() // UniqueIdentifier ); jdt->ditype = ct; std::vector Elements; From 69a605bad0d0e0be00eb545a14e1385a732533e4 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 20 Aug 2016 11:08:00 -0500 Subject: [PATCH 108/114] reshape(::Array, Val{N}) always returns an Array (cherry picked from commit a3e6fcf138f5cc849eaa0ea626f53e8480270137) ref #18160 reshape: only call to_shape when it will return Dims (cherry picked from commit d92b2db4cc215f4568532876fae2e93fbe8bec6f) --- base/reshapedarray.jl | 31 +++++++++++++++++++++---------- test/arrayops.jl | 8 ++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index 0f82bcca73d64..fa00f37b0001b 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -36,8 +36,9 @@ start(R::ReshapedArrayIterator) = start(R.iter) end length(R::ReshapedArrayIterator) = length(R.iter) -reshape(parent::AbstractArray, shp::Tuple) = _reshape(parent, to_shape(shp)) reshape(parent::AbstractArray, dims::IntOrInd...) = reshape(parent, dims) +reshape(parent::AbstractArray, shp::NeedsShaping) = reshape(parent, to_shape(shp)) +reshape(parent::AbstractArray, dims::Dims) = _reshape(parent, dims) reshape{T,N}(parent::AbstractArray{T,N}, ndims::Type{Val{N}}) = parent function reshape{T,AN,N}(parent::AbstractArray{T,AN}, ndims::Type{Val{N}}) @@ -47,24 +48,34 @@ end # dimensionality N, either filling with OneTo(1) or collapsing the # product of trailing dims into the last element @pure rdims{N}(out::NTuple{N}, inds::Tuple{}, ::Type{Val{N}}) = out -@pure rdims{N}(out::NTuple{N}, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = (front(out)..., length(last(out)) * prod(map(length, inds))) +@pure function rdims{N}(out::NTuple{N}, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) + l = length(last(out)) * prod(map(length, inds)) + (front(out)..., OneTo(l)) +end @pure rdims{N}(out::Tuple, inds::Tuple{}, ::Type{Val{N}}) = rdims((out..., OneTo(1)), (), Val{N}) @pure rdims{N}(out::Tuple, inds::Tuple{Any, Vararg{Any}}, ::Type{Val{N}}) = rdims((out..., first(inds)), tail(inds), Val{N}) -function _reshape(parent::AbstractArray, dims::Dims) - n = _length(parent) - prod(dims) == n || throw(DimensionMismatch("parent has $n elements, which is incompatible with size $dims")) - __reshape((parent, linearindexing(parent)), dims) -end -_reshape(R::ReshapedArray, dims::Dims) = _reshape(R.parent, dims) +# _reshape on Array returns an Array +_reshape(parent::Vector, dims::Dims{1}) = parent +_reshape(parent::Array, dims::Dims{1}) = reshape(parent, dims) +_reshape(parent::Array, dims::Dims) = reshape(parent, dims) # When reshaping Vector->Vector, don't wrap with a ReshapedArray -_reshape{T}(v::ReshapedArray{T,1}, dims::Tuple{Int}) = _reshape(v.parent, dims) -function _reshape(v::AbstractVector, dims::Tuple{Int}) +function _reshape(v::AbstractVector, dims::Dims{1}) len = dims[1] len == length(v) || throw(DimensionMismatch("parent has $(length(v)) elements, which is incompatible with length $len")) v end +# General reshape +function _reshape(parent::AbstractArray, dims::Dims) + n = _length(parent) + prod(dims) == n || throw(DimensionMismatch("parent has $n elements, which is incompatible with size $dims")) + __reshape((parent, linearindexing(parent)), dims) +end + +# Reshaping a ReshapedArray +_reshape{T}(v::ReshapedArray{T,1}, dims::Dims{1}) = _reshape(v.parent, dims) +_reshape(R::ReshapedArray, dims::Dims) = _reshape(R.parent, dims) function __reshape(p::Tuple{AbstractArray,LinearSlow}, dims::Dims) parent = p[1] diff --git a/test/arrayops.jl b/test/arrayops.jl index 5280484c95faf..7a68a0492af76 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -124,6 +124,14 @@ a = zeros(0, 5) # an empty linearslow array s = view(a, :, [2,3,5]) @test length(reshape(s, length(s))) == 0 +# reshape(a, Val{N}) +a = ones(Int,3,3) +s = view(a, 1:2, 1:2) +for N in (1,3) + @test isa(reshape(a, Val{N}), Array{Int,N}) + @test isa(reshape(s, Val{N}), Base.ReshapedArray{Int,N}) +end + @test reshape(1:5, (5,)) === 1:5 @test reshape(1:5, 5) === 1:5 From fafdc7b8a3d365f03325f8fbb145e4ccf883c85e Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 21 Aug 2016 19:29:10 -0400 Subject: [PATCH 109/114] Increase MEMDEBUG allocation threshold for #14173 test (cherry picked from commit 8e96740236ae6da65f94335bd17139bd2f7884ab) ref #18171 --- test/misc.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/misc.jl b/test/misc.jl index e91c3ca135b8c..114c0f2b9079f 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -231,7 +231,7 @@ module Tmp14173 end whos(IOBuffer(), Tmp14173) # warm up const MEMDEBUG = ccall(:jl_is_memdebug, Bool, ()) -@test @allocated(whos(IOBuffer(), Tmp14173)) < (MEMDEBUG ? 20000 : 8000) +@test @allocated(whos(IOBuffer(), Tmp14173)) < (MEMDEBUG ? 30000 : 8000) ## test conversion from UTF-8 to UTF-16 (for Windows APIs) From d45bcaeb76c2693834520f8f39785903ab85efd3 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Mon, 22 Aug 2016 04:51:15 -0400 Subject: [PATCH 110/114] Revert "fix #18129, optimize some more cases of captured variables" This reverts commit 09f7005d48e3390333cfd2d381b6b34aa5a5bece. un-backports #18130 for now until there's a fix for #18173 (was breaking BaseTestDeprecated.jl) --- src/julia-syntax.scm | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index aaf07bca7d74a..f762753a5f80f 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2721,27 +2721,25 @@ f(x) = yt(x) (any vinfo:sa vi)) (let* ((leading (filter (lambda (x) (and (pair? x) - (let ((cx (car x))) - (or (and (eq? cx 'method) (length> x 2)) - (eq? cx '=) - (eq? cx 'call))))) + (or (and (eq? (car x) 'method) + (length> x 2)) + (eq? (car x) '=)))) (take-statements-while (lambda (e) (or (atom? e) (memq (car e) '(quote top core line inert local unnecessary meta inbounds boundscheck simdloop implicit-global global globalref - const newvar = null method call)))) + const newvar = null method)))) (lam:body lam)))) - (unused (map cadr (filter (lambda (x) (memq (car x) '(method =))) - leading))) + (unused (map cadr leading)) (def (table))) ;; TODO: reorder leading statements to put assignments where the RHS is ;; `simple-atom?` at the top. (for-each (lambda (e) (set! unused (filter (lambda (v) (not (expr-uses-var e v))) unused)) - (if (and (memq (car e) '(method =)) (memq (cadr e) unused)) + (if (memq (cadr e) unused) (put! def (cadr e) #t))) leading) (for-each (lambda (v) From f06e0f1271dddaeeaf4f8782f8d5ea0a971f9b75 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 16 Aug 2016 00:29:05 -0400 Subject: [PATCH 111/114] fall back to dynamic dispatch in union-splitting optimization This should help #17932. If a function returns an unanticipated type that wasn't part of the `Union` seen by inference, do dynamic dispatch instead of erroring. (cherry picked from commit b0cf59170ceb4032c8a01c65b4f21ca878f566be) ref #18046 --- base/inference.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 79edb33b96cbd..2649514a32bd1 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -9,7 +9,8 @@ const MAX_TUPLETYPE_LEN = 15 const MAX_TUPLE_DEPTH = 4 const MAX_TUPLE_SPLAT = 16 -const MAX_UNION_SPLITTING = 6 +const MAX_UNION_SPLITTING = 4 +const UNION_SPLIT_MISMATCH_ERROR = false # alloc_elim_pass! relies on `Slot_AssignedOnce | Slot_UsedUndef` being # SSA. This should be true now but can break if we start to track conditional @@ -2437,7 +2438,7 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference all = false end end - if all + if UNION_SPLIT_MISMATCH_ERROR && all error_label === nothing && (error_label = genlabel(sv)) push!(stmts, GotoNode(error_label.label)) else From 2406ed91a2f0ab9e81b21f30eba769816b7d4553 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 22 Aug 2016 07:42:00 -0500 Subject: [PATCH 112/114] Fix fft(::Vector{Complex{Int}}) (cherry picked from commit e4ea3a7e782e2ea21da64a14f7dfcf862342d18a) ref #18178 --- base/dft.jl | 1 + test/fft.jl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/base/dft.jl b/base/dft.jl index 700bbc68c2dec..f1072d2ae99e6 100644 --- a/base/dft.jl +++ b/base/dft.jl @@ -24,6 +24,7 @@ typealias FFTWFloat Union{Float32,Float64} fftwfloat(x) = _fftwfloat(float(x)) _fftwfloat{T<:FFTWFloat}(::Type{T}) = T _fftwfloat(::Type{Float16}) = Float32 +_fftwfloat{T}(::Type{Complex{T}}) = Complex{_fftwfloat(T)} _fftwfloat{T}(::Type{T}) = error("type $T not supported") _fftwfloat{T}(x::T) = _fftwfloat(T)(x) diff --git a/test/fft.jl b/test/fft.jl index b30e05ff09a5a..376d7e5e32248 100644 --- a/test/fft.jl +++ b/test/fft.jl @@ -6,6 +6,8 @@ a = rand(8) + im*rand(8) @test norm(ifft(fft(a,1),1) - a) < 1e-8 @test norm(ifft(fft(a,[1]),[1]) - a) < 1e-8 @test norm(ifft(fft(a,(1,)),(1,)) - a) < 1e-8 +a = rand(-10:10, 8) + im*rand(-10:10, 8) +@test norm(ifft(fft(a)) - a) < 1e-8 m4 = [16. 2 3 13; 5 11 10 8; From fe17200dc305e6b1e1b0e2ea54135e7f04bb4d40 Mon Sep 17 00:00:00 2001 From: Tony Kelman Date: Mon, 22 Aug 2016 09:35:01 -0700 Subject: [PATCH 113/114] Revert "Revert "fix #18129, optimize some more cases of captured variables"" This reverts commit d45bcaeb76c2693834520f8f39785903ab85efd3. --- src/julia-syntax.scm | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index f762753a5f80f..aaf07bca7d74a 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2721,25 +2721,27 @@ f(x) = yt(x) (any vinfo:sa vi)) (let* ((leading (filter (lambda (x) (and (pair? x) - (or (and (eq? (car x) 'method) - (length> x 2)) - (eq? (car x) '=)))) + (let ((cx (car x))) + (or (and (eq? cx 'method) (length> x 2)) + (eq? cx '=) + (eq? cx 'call))))) (take-statements-while (lambda (e) (or (atom? e) (memq (car e) '(quote top core line inert local unnecessary meta inbounds boundscheck simdloop implicit-global global globalref - const newvar = null method)))) + const newvar = null method call)))) (lam:body lam)))) - (unused (map cadr leading)) + (unused (map cadr (filter (lambda (x) (memq (car x) '(method =))) + leading))) (def (table))) ;; TODO: reorder leading statements to put assignments where the RHS is ;; `simple-atom?` at the top. (for-each (lambda (e) (set! unused (filter (lambda (v) (not (expr-uses-var e v))) unused)) - (if (memq (cadr e) unused) + (if (and (memq (car e) '(method =)) (memq (cadr e) unused)) (put! def (cadr e) #t))) leading) (for-each (lambda (v) From e5b833266c3d17d82c00584bd235fa773d9a4905 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 22 Aug 2016 12:05:08 -0400 Subject: [PATCH 114/114] fix #18173, closure lowering error caused by #18130 (cherry picked from commit ca5973ab56e0e212ba8b0e57266225ef28a1dcaa) ref #18183 --- src/julia-syntax.scm | 2 +- test/core.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index aaf07bca7d74a..c8837d90c6d41 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2714,7 +2714,7 @@ f(x) = yt(x) ((eq? (car ex) 'method) (and (length> ex 2) (assq v (cadr (lam:vinfo (cadddr ex)))))) - (else #f))) + (else (expr-contains-eq v ex)))) (assert (eq? (car lam) 'lambda)) (let ((vi (car (lam:vinfo lam)))) (if (and (any vinfo:capt vi) diff --git a/test/core.jl b/test/core.jl index 66ba2ef72368f..d0295b804d87b 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4484,3 +4484,10 @@ for (f,g) in ((:asin,:sin), (:acos,:cos)) f18085(::Type{Val{f}},x...) = map(x->2gx(x), f18085(Val{g},x...)) end @test f18085(Val{:asin},3) === (0.0,) + +# issue #18173 +function f18173() + identity(()->successflag) + successflag = false +end +@test f18173() == false