From aa23b4669009197fdab99ee627061a4c8e05a703 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Sun, 4 Feb 2024 20:50:07 +0100 Subject: [PATCH 01/12] Use Memory in some more fast methods This commit changes the signature of a bunch of methods that operate on byte buffers, and which previously didn't take `Memory`. For example, a method which previously took ```julia Union{ Array{UInt8}, CodeUnits{UInt8, String}, FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N, } ``` Now also takes `Memory{UInt8}`, and the `Memory`-backed `FastContiguousSubArray`. --- base/iostream.jl | 21 ++++++++++++++++++--- base/strings/string.jl | 14 +++++++++++++- base/util.jl | 11 ++++++++++- stdlib/CRC32c/src/CRC32c.jl | 12 +++++++++++- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/base/iostream.jl b/base/iostream.jl index f39d9013a7fc9..96c23f642922f 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -487,7 +487,12 @@ function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false) end function readbytes_all!(s::IOStream, - b::Union{Array{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}}, + b::Union{ + Array{UInt8}, + Memory{UInt8}, + FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, + FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, + }, nb::Integer) olb = lb = length(b) nr = 0 @@ -517,7 +522,12 @@ function readbytes_all!(s::IOStream, end function readbytes_some!(s::IOStream, - b::Union{Array{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}}, + b::Union{ + Array{UInt8}, + Memory{UInt8}, + FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, + FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, + }, nb::Integer) olb = length(b) if nb > olb @@ -548,7 +558,12 @@ requested bytes, until an error or end-of-file occurs. If `all` is `false`, at m all stream types support the `all` option. """ function readbytes!(s::IOStream, - b::Union{Array{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}}, + b::Union{ + Array{UInt8}, + Memory{UInt8}, + FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, + FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, + }, nb=length(b); all::Bool=true) return all ? readbytes_all!(s, b, nb) : readbytes_some!(s, b, nb) diff --git a/base/strings/string.jl b/base/strings/string.jl index d091baeb6c663..f1d319ea1b7c0 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -27,7 +27,19 @@ function Base.showerror(io::IO, exc::StringIndexError) end end -const ByteArray = Union{CodeUnits{UInt8,String}, Vector{UInt8},Vector{Int8}, FastContiguousSubArray{UInt8,1,CodeUnits{UInt8,String}}, FastContiguousSubArray{UInt8,1,Vector{UInt8}}, FastContiguousSubArray{Int8,1,Vector{Int8}}} +const ByteArray = Union{ + CodeUnits{UInt8,String}, + CodeUnits{UInt8,SubString{String}}, + Vector{UInt8}, + Vector{Int8}, + Memory{UInt8}, + Memory{Int8}, + FastContiguousSubArray{UInt8,1,CodeUnits{UInt8,String}}, + FastContiguousSubArray{UInt8,1,Vector{UInt8}}, + FastContiguousSubArray{Int8,1,Vector{Int8}}, + FastContiguousSubArray{UInt8,1,Memory{UInt8}}, + FastContiguousSubArray{Int8,1,Memory{Int8}}, +} @inline between(b::T, lo::T, hi::T) where {T<:Integer} = (lo ≤ b) & (b ≤ hi) diff --git a/base/util.jl b/base/util.jl index 5e86f026f8f9a..07f8f7f157a6f 100644 --- a/base/util.jl +++ b/base/util.jl @@ -491,8 +491,17 @@ unsafe_crc32c(a, n, crc) = ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_ _crc32c(a::NTuple{<:Any, UInt8}, crc::UInt32=0x00000000) = unsafe_crc32c(Ref(a), length(a) % Csize_t, crc) -_crc32c(a::Union{Array{UInt8},FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N}, crc::UInt32=0x00000000) = + +function _crc32c(a::Union{ + Array{UInt8}, + Memory{UInt8}, + FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N, + FastContiguousSubArray{UInt8,N,<:Memory{UInt8}} where N, + }, + crc::UInt32=0x00000000 +) unsafe_crc32c(a, length(a) % Csize_t, crc) +end function _crc32c(s::Union{String, SubString{String}}, crc::UInt32=0x00000000) unsafe_crc32c(s, sizeof(s) % Csize_t, crc) diff --git a/stdlib/CRC32c/src/CRC32c.jl b/stdlib/CRC32c/src/CRC32c.jl index 35d2d4cb339d6..ff668b3aaeed7 100644 --- a/stdlib/CRC32c/src/CRC32c.jl +++ b/stdlib/CRC32c/src/CRC32c.jl @@ -35,7 +35,17 @@ but note that the result may be endian-dependent. function crc32c end -crc32c(a::Union{Array{UInt8},FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N}, crc::UInt32=0x00000000) = Base._crc32c(a, crc) +function crc32c(a::Union{ + Array{UInt8}, + Memory{UInt8}, + FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N, + FastContiguousSubArray{UInt8,N,<:Memory{UInt8}} where N, + }, + crc::UInt32=0x00000000 +) + Base._crc32c(a, crc) +end + crc32c(s::Union{String, SubString{String}}, crc::UInt32=0x00000000) = Base._crc32c(s, crc) """ From d08b4b25e17fdb956b6c8e92abb8f964f480ed3a Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 5 Feb 2024 08:48:20 +0100 Subject: [PATCH 02/12] Simplify types using DenseArray --- base/iostream.jl | 21 +++------------------ base/strings/string.jl | 13 ++----------- base/util.jl | 8 ++------ stdlib/CRC32c/src/CRC32c.jl | 8 ++------ 4 files changed, 9 insertions(+), 41 deletions(-) diff --git a/base/iostream.jl b/base/iostream.jl index 96c23f642922f..1e5b444915a9e 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -487,12 +487,7 @@ function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false) end function readbytes_all!(s::IOStream, - b::Union{ - Array{UInt8}, - Memory{UInt8}, - FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, - FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, - }, + b::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:DenseArray{UInt8}}}, nb::Integer) olb = lb = length(b) nr = 0 @@ -522,12 +517,7 @@ function readbytes_all!(s::IOStream, end function readbytes_some!(s::IOStream, - b::Union{ - Array{UInt8}, - Memory{UInt8}, - FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, - FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, - }, + b::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:DenseArray{UInt8}}}, nb::Integer) olb = length(b) if nb > olb @@ -558,12 +548,7 @@ requested bytes, until an error or end-of-file occurs. If `all` is `false`, at m all stream types support the `all` option. """ function readbytes!(s::IOStream, - b::Union{ - Array{UInt8}, - Memory{UInt8}, - FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, - FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, - }, + b::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:DenseArray{UInt8}}}, nb=length(b); all::Bool=true) return all ? readbytes_all!(s, b, nb) : readbytes_some!(s, b, nb) diff --git a/base/strings/string.jl b/base/strings/string.jl index f1d319ea1b7c0..cffd18c9271e2 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -28,17 +28,8 @@ function Base.showerror(io::IO, exc::StringIndexError) end const ByteArray = Union{ - CodeUnits{UInt8,String}, - CodeUnits{UInt8,SubString{String}}, - Vector{UInt8}, - Vector{Int8}, - Memory{UInt8}, - Memory{Int8}, - FastContiguousSubArray{UInt8,1,CodeUnits{UInt8,String}}, - FastContiguousSubArray{UInt8,1,Vector{UInt8}}, - FastContiguousSubArray{Int8,1,Vector{Int8}}, - FastContiguousSubArray{UInt8,1,Memory{UInt8}}, - FastContiguousSubArray{Int8,1,Memory{Int8}}, + DenseVector{<:Union{UInt8, Int8}}, + FastContiguousSubArray{<:Union{UInt8, Int8},1,<:DenseVector{<:Union{UInt8, Int8}}}, } @inline between(b::T, lo::T, hi::T) where {T<:Integer} = (lo ≤ b) & (b ≤ hi) diff --git a/base/util.jl b/base/util.jl index 07f8f7f157a6f..aa94d5a968206 100644 --- a/base/util.jl +++ b/base/util.jl @@ -492,12 +492,8 @@ unsafe_crc32c(a, n, crc) = ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_ _crc32c(a::NTuple{<:Any, UInt8}, crc::UInt32=0x00000000) = unsafe_crc32c(Ref(a), length(a) % Csize_t, crc) -function _crc32c(a::Union{ - Array{UInt8}, - Memory{UInt8}, - FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N, - FastContiguousSubArray{UInt8,N,<:Memory{UInt8}} where N, - }, +function _crc32c( + a::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,N,<:DenseArray{UInt8}} where N}, crc::UInt32=0x00000000 ) unsafe_crc32c(a, length(a) % Csize_t, crc) diff --git a/stdlib/CRC32c/src/CRC32c.jl b/stdlib/CRC32c/src/CRC32c.jl index ff668b3aaeed7..3e430e699f5b9 100644 --- a/stdlib/CRC32c/src/CRC32c.jl +++ b/stdlib/CRC32c/src/CRC32c.jl @@ -35,12 +35,8 @@ but note that the result may be endian-dependent. function crc32c end -function crc32c(a::Union{ - Array{UInt8}, - Memory{UInt8}, - FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N, - FastContiguousSubArray{UInt8,N,<:Memory{UInt8}} where N, - }, +function crc32c( + a::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,N,<:DenseArray{UInt8}} where N}, crc::UInt32=0x00000000 ) Base._crc32c(a, crc) From a8ca085056756170a14f3b38a5a8d58905f2a831 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 5 Feb 2024 08:55:00 +0100 Subject: [PATCH 03/12] Do not include codeunits in io --- base/iostream.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/base/iostream.jl b/base/iostream.jl index 1e5b444915a9e..49a6803993ee8 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -486,9 +486,14 @@ function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false) return out end -function readbytes_all!(s::IOStream, - b::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:DenseArray{UInt8}}}, - nb::Integer) +const MutableByteArray = Union{ + Array{UInt8}, + Memory{UInt8}, + FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}}, + FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}} +} + +function readbytes_all!(s::IOStream, b::MutableByteArray, nb::Integer) olb = lb = length(b) nr = 0 let l = s._dolock, slock = s.lock @@ -516,9 +521,7 @@ function readbytes_all!(s::IOStream, return nr end -function readbytes_some!(s::IOStream, - b::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:DenseArray{UInt8}}}, - nb::Integer) +function readbytes_some!(s::IOStream, b::MutableByteArray, nb::Integer) olb = length(b) if nb > olb resize!(b, nb) @@ -547,10 +550,7 @@ requested bytes, until an error or end-of-file occurs. If `all` is `false`, at m `read` call is performed, and the amount of data returned is device-dependent. Note that not all stream types support the `all` option. """ -function readbytes!(s::IOStream, - b::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,<:Any,<:DenseArray{UInt8}}}, - nb=length(b); - all::Bool=true) +function readbytes!(s::IOStream, b::MutableByteArray, nb=length(b); all::Bool=true) return all ? readbytes_all!(s, b, nb) : readbytes_some!(s, b, nb) end From 49e58721706a68d62b1079712bfe435a8da4a5aa Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 5 Feb 2024 08:57:40 +0100 Subject: [PATCH 04/12] Typo --- base/iostream.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/iostream.jl b/base/iostream.jl index 49a6803993ee8..d322bea6ead60 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -489,8 +489,8 @@ end const MutableByteArray = Union{ Array{UInt8}, Memory{UInt8}, - FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}}, - FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}} + FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, + FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, } function readbytes_all!(s::IOStream, b::MutableByteArray, nb::Integer) From d62749181793bd1977fe0cf12a1d2b2b36aa3433 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 5 Feb 2024 16:18:29 +0100 Subject: [PATCH 05/12] Add a few more methods --- base/filesystem.jl | 2 +- base/iobuffer.jl | 11 +++++++++-- base/iostream.jl | 7 ------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/base/filesystem.jl b/base/filesystem.jl index 36e4bb4596285..74f54ee270feb 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -309,7 +309,7 @@ bytesavailable(f::File) = max(0, filesize(f) - position(f)) # position can be > eof(f::File) = bytesavailable(f) == 0 -function readbytes!(f::File, b::Array{UInt8}, nb=length(b)) +function readbytes!(f::File, b::MutableByteArray, nb=length(b)) nr = min(nb, bytesavailable(f)) if length(b) < nr resize!(b, nr) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index f9585b0599919..b687a3203617e 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -548,8 +548,15 @@ end return sizeof(UInt8) end -readbytes!(io::GenericIOBuffer, b::Array{UInt8}, nb=length(b)) = readbytes!(io, b, Int(nb)) -function readbytes!(io::GenericIOBuffer, b::Array{UInt8}, nb::Int) +const MutableByteArray = Union{ + Array{UInt8}, + Memory{UInt8}, + FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, + FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, +} + +readbytes!(io::GenericIOBuffer, b::MutableByteArray, nb=length(b)) = readbytes!(io, b, Int(nb)) +function readbytes!(io::GenericIOBuffer, b::MutableByteArray, nb::Int) nr = min(nb, bytesavailable(io)) if length(b) < nr resize!(b, nr) diff --git a/base/iostream.jl b/base/iostream.jl index d322bea6ead60..403842c9fc6bd 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -486,13 +486,6 @@ function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false) return out end -const MutableByteArray = Union{ - Array{UInt8}, - Memory{UInt8}, - FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, - FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, -} - function readbytes_all!(s::IOStream, b::MutableByteArray, nb::Integer) olb = lb = length(b) nr = 0 From f5bee85e45d164341876fb1b05e280050866ced3 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 5 Feb 2024 16:23:51 +0100 Subject: [PATCH 06/12] Import MutableByteArray to Filesystem --- base/filesystem.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/filesystem.jl b/base/filesystem.jl index 74f54ee270feb..929808b02b4f0 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -141,7 +141,7 @@ import .Base: bytesavailable, position, read, read!, readavailable, seek, seekend, show, skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error, setup_stdio, rawhandle, OS_HANDLE, INVALID_OS_HANDLE, windowserror, filesize, - isexecutable, isreadable, iswritable + isexecutable, isreadable, iswritable, MutableByteArray import .Base.RefValue From fc44874ad5520d4550b6278cbb292b4bf132be6c Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 5 Feb 2024 16:36:21 +0100 Subject: [PATCH 07/12] Fix missing branch --- base/iobuffer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index b687a3203617e..f1a1238d841bb 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -219,7 +219,7 @@ function read_sub(from::GenericIOBuffer, a::AbstractArray{T}, offs, nel) where T if offs+nel-1 > length(a) || offs < 1 || nel < 0 throw(BoundsError()) end - if isbitstype(T) && isa(a,Array) + if isbitstype(T) && isa(a,MutableByteArray) nb = UInt(nel * sizeof(T)) GC.@preserve a unsafe_read(from, pointer(a, offs), nb) else From 9e3e7b758706b0fb35d1ed454ca770d30849e972 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Sat, 25 May 2024 14:18:30 +0200 Subject: [PATCH 08/12] New definition --- base/filesystem.jl | 4 ++-- base/iobuffer.jl | 13 +++---------- base/iostream.jl | 6 +++--- base/strings/string.jl | 5 ----- base/subarray.jl | 18 ++++++++++++++++++ base/util.jl | 5 +---- stdlib/CRC32c/src/CRC32c.jl | 6 ++---- 7 files changed, 29 insertions(+), 28 deletions(-) diff --git a/base/filesystem.jl b/base/filesystem.jl index 929808b02b4f0..55f16dda51e0d 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -141,7 +141,7 @@ import .Base: bytesavailable, position, read, read!, readavailable, seek, seekend, show, skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error, setup_stdio, rawhandle, OS_HANDLE, INVALID_OS_HANDLE, windowserror, filesize, - isexecutable, isreadable, iswritable, MutableByteArray + isexecutable, isreadable, iswritable, MutableDenseArrayType import .Base.RefValue @@ -309,7 +309,7 @@ bytesavailable(f::File) = max(0, filesize(f) - position(f)) # position can be > eof(f::File) = bytesavailable(f) == 0 -function readbytes!(f::File, b::MutableByteArray, nb=length(b)) +function readbytes!(f::File, b::MutableDenseArrayType{UInt8}, nb=length(b)) nr = min(nb, bytesavailable(f)) if length(b) < nr resize!(b, nr) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index f1a1238d841bb..04a694a4fec15 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -219,7 +219,7 @@ function read_sub(from::GenericIOBuffer, a::AbstractArray{T}, offs, nel) where T if offs+nel-1 > length(a) || offs < 1 || nel < 0 throw(BoundsError()) end - if isbitstype(T) && isa(a,MutableByteArray) + if isa(a, MutableDenseArrayType{UInt8}) nb = UInt(nel * sizeof(T)) GC.@preserve a unsafe_read(from, pointer(a, offs), nb) else @@ -548,15 +548,8 @@ end return sizeof(UInt8) end -const MutableByteArray = Union{ - Array{UInt8}, - Memory{UInt8}, - FastContiguousSubArray{UInt8,<:Any,<:Array{UInt8}}, - FastContiguousSubArray{UInt8,<:Any,<:Memory{UInt8}}, -} - -readbytes!(io::GenericIOBuffer, b::MutableByteArray, nb=length(b)) = readbytes!(io, b, Int(nb)) -function readbytes!(io::GenericIOBuffer, b::MutableByteArray, nb::Int) +readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb=length(b)) = readbytes!(io, b, Int(nb)) +function readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb::Int) nr = min(nb, bytesavailable(io)) if length(b) < nr resize!(b, nr) diff --git a/base/iostream.jl b/base/iostream.jl index 403842c9fc6bd..1a89984fc96c7 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -486,7 +486,7 @@ function copyuntil(out::IOStream, s::IOStream, delim::UInt8; keep::Bool=false) return out end -function readbytes_all!(s::IOStream, b::MutableByteArray, nb::Integer) +function readbytes_all!(s::IOStream, b::MutableDenseArrayType{UInt8}, nb::Integer) olb = lb = length(b) nr = 0 let l = s._dolock, slock = s.lock @@ -514,7 +514,7 @@ function readbytes_all!(s::IOStream, b::MutableByteArray, nb::Integer) return nr end -function readbytes_some!(s::IOStream, b::MutableByteArray, nb::Integer) +function readbytes_some!(s::IOStream, b::MutableDenseArrayType{UInt8}, nb::Integer) olb = length(b) if nb > olb resize!(b, nb) @@ -543,7 +543,7 @@ requested bytes, until an error or end-of-file occurs. If `all` is `false`, at m `read` call is performed, and the amount of data returned is device-dependent. Note that not all stream types support the `all` option. """ -function readbytes!(s::IOStream, b::MutableByteArray, nb=length(b); all::Bool=true) +function readbytes!(s::IOStream, b::MutableDenseArrayType{UInt8}, nb=length(b); all::Bool=true) return all ? readbytes_all!(s, b, nb) : readbytes_some!(s, b, nb) end diff --git a/base/strings/string.jl b/base/strings/string.jl index cffd18c9271e2..cfe2bc8adfdac 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -27,11 +27,6 @@ function Base.showerror(io::IO, exc::StringIndexError) end end -const ByteArray = Union{ - DenseVector{<:Union{UInt8, Int8}}, - FastContiguousSubArray{<:Union{UInt8, Int8},1,<:DenseVector{<:Union{UInt8, Int8}}}, -} - @inline between(b::T, lo::T, hi::T) where {T<:Integer} = (lo ≤ b) & (b ≤ hi) """ diff --git a/base/subarray.jl b/base/subarray.jl index e4bf5579ff90e..7557772033b9e 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -348,6 +348,24 @@ FastContiguousSubArray{T,N,P,I<:Union{Tuple{Union{Slice, AbstractUnitRange}, Var @inline _reindexlinear(V::FastContiguousSubArray, i::Int) = V.offset1 + i @inline _reindexlinear(V::FastContiguousSubArray, i::AbstractUnitRange{Int}) = V.offset1 .+ i +""" +An internal type representing arrays stored contiguously in memory. +""" +const DenseArrayType{T,N} = Union{ + DenseArray{T,N}, + <:FastContiguousSubArray{T,N,<:DenseArray}, +} + +""" +An internal type representing mutable arrays stored contiguously in memory. +""" +const MutableDenseArrayType{T,N} = Union{ + Array{T, N}, + Memory{T}, + FastContiguousSubArray{T,N,<:Array}, + FastContiguousSubArray{T,N,<:Memory} +} + # parents of FastContiguousSubArrays may support fast indexing with AbstractUnitRanges, # so we may just forward the indexing to the parent # This may only be done for non-offset ranges, as the result would otherwise have offset axes diff --git a/base/util.jl b/base/util.jl index aa94d5a968206..1cb0beca9dbde 100644 --- a/base/util.jl +++ b/base/util.jl @@ -492,10 +492,7 @@ unsafe_crc32c(a, n, crc) = ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_ _crc32c(a::NTuple{<:Any, UInt8}, crc::UInt32=0x00000000) = unsafe_crc32c(Ref(a), length(a) % Csize_t, crc) -function _crc32c( - a::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,N,<:DenseArray{UInt8}} where N}, - crc::UInt32=0x00000000 -) +function _crc32c(a::DenseArrayType{UInt8}, crc::UInt32=0x00000000) unsafe_crc32c(a, length(a) % Csize_t, crc) end diff --git a/stdlib/CRC32c/src/CRC32c.jl b/stdlib/CRC32c/src/CRC32c.jl index 3e430e699f5b9..08435f6c4bbca 100644 --- a/stdlib/CRC32c/src/CRC32c.jl +++ b/stdlib/CRC32c/src/CRC32c.jl @@ -8,6 +8,7 @@ See [`CRC32c.crc32c`](@ref) for more information. module CRC32c import Base.FastContiguousSubArray +import Base: DenseArrayType export crc32c @@ -35,10 +36,7 @@ but note that the result may be endian-dependent. function crc32c end -function crc32c( - a::Union{DenseArray{UInt8}, FastContiguousSubArray{UInt8,N,<:DenseArray{UInt8}} where N}, - crc::UInt32=0x00000000 -) +function crc32c(a::DenseArrayType{UInt8}, crc::UInt32=0x00000000) Base._crc32c(a, crc) end From 5f73f8a3325df10eda812b7b03d1cb55ac33a9bb Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Sat, 25 May 2024 14:47:48 +0200 Subject: [PATCH 09/12] Fixup ByteArray --- base/strings/search.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/strings/search.jl b/base/strings/search.jl index e2b3dc96b98cf..d46b7f9a3d57e 100644 --- a/base/strings/search.jl +++ b/base/strings/search.jl @@ -29,6 +29,8 @@ function findnext(pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:AbstractChar} end end +const ByteArray = Union{DenseArrayType{UInt8}, DenseArrayType{Int8}} + findfirst(pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:Union{Int8,UInt8}}, a::ByteArray) = nothing_sentinel(_search(a, pred.x)) @@ -38,7 +40,7 @@ findnext(pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:Union{Int8,UInt8}}, a: findfirst(::typeof(iszero), a::ByteArray) = nothing_sentinel(_search(a, zero(UInt8))) findnext(::typeof(iszero), a::ByteArray, i::Integer) = nothing_sentinel(_search(a, zero(UInt8), i)) -function _search(a::Union{String,SubString{String},ByteArray}, b::Union{Int8,UInt8}, i::Integer = 1) +function _search(a::Union{String,SubString{String},<:ByteArray}, b::Union{Int8,UInt8}, i::Integer = 1) if i < 1 throw(BoundsError(a, i)) end From 929cf107075d2add6aff5f6df5c82c945d38b82c Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Sun, 26 May 2024 09:23:37 +0200 Subject: [PATCH 10/12] Include codeunits with a new DenseBytes type --- base/strings/search.jl | 2 +- base/subarray.jl | 8 ++++++++ base/util.jl | 2 +- stdlib/CRC32c/src/CRC32c.jl | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/base/strings/search.jl b/base/strings/search.jl index d46b7f9a3d57e..1cf480611613b 100644 --- a/base/strings/search.jl +++ b/base/strings/search.jl @@ -29,7 +29,7 @@ function findnext(pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:AbstractChar} end end -const ByteArray = Union{DenseArrayType{UInt8}, DenseArrayType{Int8}} +const ByteArray = Union{DenseBytes, DenseArrayType{Int8}} findfirst(pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:Union{Int8,UInt8}}, a::ByteArray) = nothing_sentinel(_search(a, pred.x)) diff --git a/base/subarray.jl b/base/subarray.jl index 7557772033b9e..f15dfd12dcd25 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -366,6 +366,14 @@ const MutableDenseArrayType{T,N} = Union{ FastContiguousSubArray{T,N,<:Memory} } +# Note: Currently, CodeUnits <: DenseVector, which makes this union redundant w.r.t +# DenseArrayType{UInt8}, but this is a bug, and may be removed in future versions +# of Julia. See #54002 +const DenseBytes = Union{ + <:DenseArrayType{UInt8}, + CodeUnits{UInt8, <:Union{String, SubString{String}}}, +} + # parents of FastContiguousSubArrays may support fast indexing with AbstractUnitRanges, # so we may just forward the indexing to the parent # This may only be done for non-offset ranges, as the result would otherwise have offset axes diff --git a/base/util.jl b/base/util.jl index 1cb0beca9dbde..e9883650abc77 100644 --- a/base/util.jl +++ b/base/util.jl @@ -492,7 +492,7 @@ unsafe_crc32c(a, n, crc) = ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_ _crc32c(a::NTuple{<:Any, UInt8}, crc::UInt32=0x00000000) = unsafe_crc32c(Ref(a), length(a) % Csize_t, crc) -function _crc32c(a::DenseArrayType{UInt8}, crc::UInt32=0x00000000) +function _crc32c(a::DenseBytes, crc::UInt32=0x00000000) unsafe_crc32c(a, length(a) % Csize_t, crc) end diff --git a/stdlib/CRC32c/src/CRC32c.jl b/stdlib/CRC32c/src/CRC32c.jl index 08435f6c4bbca..577505f1c57cb 100644 --- a/stdlib/CRC32c/src/CRC32c.jl +++ b/stdlib/CRC32c/src/CRC32c.jl @@ -36,7 +36,7 @@ but note that the result may be endian-dependent. function crc32c end -function crc32c(a::DenseArrayType{UInt8}, crc::UInt32=0x00000000) +function crc32c(a::DenseBytes, crc::UInt32=0x00000000) Base._crc32c(a, crc) end From dbcfe7501fd9e35d3f62da5a51f913798dd48510 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Sun, 26 May 2024 11:01:44 +0200 Subject: [PATCH 11/12] Move DenseBytes --- base/strings/search.jl | 8 ++++++++ base/subarray.jl | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/base/strings/search.jl b/base/strings/search.jl index 1cf480611613b..493e241723867 100644 --- a/base/strings/search.jl +++ b/base/strings/search.jl @@ -29,6 +29,14 @@ function findnext(pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:AbstractChar} end end +# Note: Currently, CodeUnits <: DenseVector, which makes this union redundant w.r.t +# DenseArrayType{UInt8}, but this is a bug, and may be removed in future versions +# of Julia. See #54002 +const DenseBytes = Union{ + <:DenseArrayType{UInt8}, + CodeUnits{UInt8, <:Union{String, SubString{String}}}, +} + const ByteArray = Union{DenseBytes, DenseArrayType{Int8}} findfirst(pred::Fix2{<:Union{typeof(isequal),typeof(==)},<:Union{Int8,UInt8}}, a::ByteArray) = diff --git a/base/subarray.jl b/base/subarray.jl index f15dfd12dcd25..7557772033b9e 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -366,14 +366,6 @@ const MutableDenseArrayType{T,N} = Union{ FastContiguousSubArray{T,N,<:Memory} } -# Note: Currently, CodeUnits <: DenseVector, which makes this union redundant w.r.t -# DenseArrayType{UInt8}, but this is a bug, and may be removed in future versions -# of Julia. See #54002 -const DenseBytes = Union{ - <:DenseArrayType{UInt8}, - CodeUnits{UInt8, <:Union{String, SubString{String}}}, -} - # parents of FastContiguousSubArrays may support fast indexing with AbstractUnitRanges, # so we may just forward the indexing to the parent # This may only be done for non-offset ranges, as the result would otherwise have offset axes From bb08ee1b69c9b8300a375dd797aaf679dfe7962d Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Sun, 26 May 2024 12:56:54 +0200 Subject: [PATCH 12/12] Import DenseBytes to CRC32c --- stdlib/CRC32c/src/CRC32c.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/CRC32c/src/CRC32c.jl b/stdlib/CRC32c/src/CRC32c.jl index 577505f1c57cb..03bef027bde16 100644 --- a/stdlib/CRC32c/src/CRC32c.jl +++ b/stdlib/CRC32c/src/CRC32c.jl @@ -8,7 +8,7 @@ See [`CRC32c.crc32c`](@ref) for more information. module CRC32c import Base.FastContiguousSubArray -import Base: DenseArrayType +import Base: DenseBytes export crc32c