Skip to content
This repository was archived by the owner on May 4, 2019. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion REQUIRE
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
julia 0.4-
Compat
Compat 0.7.14
Reexport
51 changes: 26 additions & 25 deletions perf/reduce.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using NullableArrays
using DataArrays
import Compat: @functorize

srand(1)
A = rand(5_000_000)
Expand Down Expand Up @@ -226,68 +227,68 @@ end
function profile_skip(skip::Bool)
println("Comparison of skipnull/skipNA methods")
println()
println("f := IdFun(), op := AddFun()")
println("f := identity, op := +")
println("mapreduce(f, op, X; skipnull/skipNA=$skip) (0 missing entries)")

mapreduce(Base.IdFun(), Base.AddFun(), X, skipnull=skip)
mapreduce(@functorize(identity), @functorize(+), X, skipnull=skip)
print(" for NullableArray{Float64}: ")
@time(mapreduce(Base.IdFun(), Base.AddFun(), X, skipnull=skip))
@time(mapreduce(@functorize(identity), @functorize(+), X, skipnull=skip))

mapreduce(Base.IdFun(), Base.AddFun(), D, skipna=skip)
mapreduce(@functorize(identity), @functorize(+), D, skipna=skip)
print(" for DataArray{Float64}: ")
@time(mapreduce(Base.IdFun(), Base.AddFun(), D, skipna=skip))
@time(mapreduce(@functorize(identity), @functorize(+), D, skipna=skip))

println()
println("reduce(op, X; skipnull/skipNA=$skip) (0 missing entries)")
reduce(Base.AddFun(), X, skipnull=skip)
reduce(@functorize(+), X, skipnull=skip)
print(" for NullableArray{Float64}: ")
@time(reduce(Base.AddFun(), X, skipnull=skip))
@time(reduce(@functorize(+), X, skipnull=skip))

reduce(Base.AddFun(), D, skipna=skip)
reduce(@functorize(+), D, skipna=skip)
print(" for DataArray{Float64}: ")
@time(reduce(Base.AddFun(), D, skipna=skip))
@time(reduce(@functorize(+), D, skipna=skip))

println()
println("mapreduce(f, op, X; skipnull/skipNA=$skip) (~half missing entries)")
mapreduce(Base.IdFun(), Base.AddFun(), Y, skipnull=skip)
mapreduce(@functorize(identity), @functorize(+), Y, skipnull=skip)
print(" for NullableArray{Float64}: ")
@time(mapreduce(Base.IdFun(), Base.AddFun(), Y, skipnull=skip))
@time(mapreduce(@functorize(identity), @functorize(+), Y, skipnull=skip))

mapreduce(Base.IdFun(), Base.AddFun(), E, skipna=skip)
mapreduce(@functorize(identity), @functorize(+), E, skipna=skip)
print(" for DataArray{Float64}: ")
@time(mapreduce(Base.IdFun(), Base.AddFun(), E, skipna=skip))
@time(mapreduce(@functorize(identity), @functorize(+), E, skipna=skip))

println()
println("reduce(op, X; skipnull/skipNA=$skip) (~half missing entries)")
reduce(Base.AddFun(), Y, skipnull=skip)
reduce(@functorize(+), Y, skipnull=skip)
print(" for NullableArray{Float64}: ")
@time(reduce(Base.AddFun(), Y, skipnull=skip))
@time(reduce(@functorize(+), Y, skipnull=skip))

reduce(Base.AddFun(), E, skipna=true)
reduce(@functorize(+), E, skipna=true)
print(" for DataArray{Float64}: ")
@time(reduce(Base.AddFun(), E, skipna=true))
@time(reduce(@functorize(+), E, skipna=true))
nothing
end

function profile_skip_impl()
println("Comparison of internal skip methods:")
println("mapreduce_impl_skipnull(f, op, X) (0 missing entries)")
NullableArrays.mapreduce_impl_skipnull(Base.IdFun(), Base.AddFun(), X)
NullableArrays.mapreduce_impl_skipnull(@functorize(identity), @functorize(+), X)
print(" for NullableArray{Float64}: ")
@time(NullableArrays.mapreduce_impl_skipnull(Base.IdFun(), Base.AddFun(), X))
@time(NullableArrays.mapreduce_impl_skipnull(@functorize(identity), @functorize(+), X))

DataArrays.mapreduce_impl_skipna(Base.IdFun(), Base.AddFun(), D)
DataArrays.mapreduce_impl_skipna(@functorize(identity), @functorize(+), D)
print(" for DataArray{Float64}: ")
@time(DataArrays.mapreduce_impl_skipna(Base.IdFun(), Base.AddFun(), D))
@time(DataArrays.mapreduce_impl_skipna(@functorize(identity), @functorize(+), D))

println()
println("mapreduce_impl_skipnull(f, op, X) (~half missing entries)")
NullableArrays.mapreduce_impl_skipnull(Base.IdFun(), Base.AddFun(), Y)
NullableArrays.mapreduce_impl_skipnull(@functorize(identity), @functorize(+), Y)
print(" for NullableArray{Float64}: ")
@time(NullableArrays.mapreduce_impl_skipnull(Base.IdFun(), Base.AddFun(), Y))
@time(NullableArrays.mapreduce_impl_skipnull(@functorize(identity), @functorize(+), Y))

DataArrays.mapreduce_impl_skipna(Base.IdFun(), Base.AddFun(), E)
DataArrays.mapreduce_impl_skipna(@functorize(identity), @functorize(+), E)
print(" for DataArray{Float64}: ")
@time(DataArrays.mapreduce_impl_skipna(Base.IdFun(), Base.AddFun(), E))
@time(DataArrays.mapreduce_impl_skipna(@functorize(identity), @functorize(+), E))
nothing
end
5 changes: 3 additions & 2 deletions src/operators.jl
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import Compat: @functorize

@noinline throw_error() = error()

Expand Down Expand Up @@ -104,14 +105,14 @@ end

## Lifted functors

@compat function (::Base.MinFun){S1, S2}(x::Nullable{S1}, y::Nullable{S2})
@compat function (::typeof(@functorize(scalarmin))){S1, S2}(x::Nullable{S1}, y::Nullable{S2})
if isbits(S1) & isbits(S2)
return Nullable(Base.scalarmin(x.value, y.value), x.isnull | y.isnull)
else
error()
end
end
@compat function (::Base.MaxFun){S1, S2}(x::Nullable{S1}, y::Nullable{S2})
@compat function (::typeof(@functorize(scalarmax))){S1, S2}(x::Nullable{S1}, y::Nullable{S2})
if isbits(S1) & isbits(S2)
return Nullable(Base.scalarmax(x.value, y.value), x.isnull | y.isnull)
else
Expand Down
66 changes: 38 additions & 28 deletions src/reduce.jl
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# interface for skipping null entries
import Compat: @functorize

function skipnull_init(f, op, X::NullableArray,
ifirst::Int, ilast::Int)
Expand Down Expand Up @@ -56,7 +57,7 @@ end

mapreduce_impl_skipnull{T}(f, op, X::NullableArray{T}) =
mapreduce_seq_impl_skipnull(f, op, X, 1, length(X.values))
mapreduce_impl_skipnull(f, op::Base.AddFun, X::NullableArray) =
mapreduce_impl_skipnull(f, op::typeof(@functorize(+)), X::NullableArray) =
mapreduce_pairwise_impl_skipnull(f, op, X, 1, length(X.values),
max(128, Base.sum_pairwise_blocksize(f)))

Expand All @@ -79,6 +80,24 @@ function Base._mapreduce(f, op, X::NullableArray, missingdata)
Nullable(Base._mapreduce(f, op, X.values))
end

# to fix ambiguity warnings
function Base.mapreduce(f, op::Union{typeof(@functorize(&)), typeof(@functorize(|))},
X::NullableArray, skipnull::Bool = false)
missingdata = anynull(X)
if skipnull
return _mapreduce_skipnull(f, op, X, missingdata)
else
return Base._mapreduce(f, op, X, missingdata)
end
end


if VERSION >= v"0.5.0-dev+3701"
const specialized_binary = identity
else
const specialized_binary = Base.specialized_binary
end

@doc """
`mapreduce(f, op::Function, X::NullableArray; [skipnull::Bool=false])`

Expand All @@ -93,21 +112,10 @@ function Base.mapreduce(f, op::Function, X::NullableArray;
skipnull::Bool = false)
missingdata = anynull(X)
if skipnull
return _mapreduce_skipnull(f, Base.specialized_binary(op),
return _mapreduce_skipnull(f, specialized_binary(op),
X, missingdata)
else
return Base._mapreduce(f, Base.specialized_binary(op), X, missingdata)
end
end

# to fix ambiguity warnings
function Base.mapreduce(f, op::Union{Base.AndFun, Base.OrFun}, X::NullableArray,
skipnull::Bool = false)
missingdata = anynull(X)
if skipnull
return _mapreduce_skipnull(f, op, X, missingdata)
else
return Base._mapreduce(f, op, X, missingdata)
return Base._mapreduce(f, specialized_binary(op), X, missingdata)
end
end

Expand All @@ -131,42 +139,44 @@ over the elements of `X`. Note that, in general, mapreducing over a
is set to `true` or `false`.
""" ->
Base.reduce(op, X::NullableArray; skipnull::Bool = false) =
mapreduce(Base.IdFun(), op, X; skipnull = skipnull)
mapreduce(@functorize(identity), op, X; skipnull = skipnull)

# standard reductions

for (fn, op) in ((:(Base.sum), Base.AddFun()),
(:(Base.prod), Base.MulFun()),
(:(Base.minimum), Base.MinFun()),
(:(Base.maximum), Base.MaxFun()))
for (fn, op) in ((:(Base.sum), @functorize(+)),
(:(Base.prod), @functorize(*)),
(:(Base.minimum), @functorize(scalarmin)),
(:(Base.maximum), @functorize(scalarmax)))
@eval begin
$fn(f::Union{Function,Base.Func{1}},
# supertype(typeof(@functorize(abs))) returns Func{1} on Julia 0.4,
# and Function on 0.5
$fn(f::Union{Function,supertype(typeof(@functorize(abs)))},
X::NullableArray;
skipnull::Bool = false) =
mapreduce(f, $op, X; skipnull = skipnull)
$fn(X::NullableArray; skipnull::Bool = false) =
mapreduce(Base.IdFun(), $op, X; skipnull = skipnull)
mapreduce(@functorize(identity), $op, X; skipnull = skipnull)
end
end

for (fn, f, op) in ((:(Base.sumabs), Base.AbsFun(), Base.AddFun()),
(:(Base.sumabs2), Base.Abs2Fun(), Base.AddFun()))
for (fn, f, op) in ((:(Base.sumabs), @functorize(abs), @functorize(+)),
(:(Base.sumabs2), @functorize(abs2), @functorize(+)))
@eval $fn(X::NullableArray; skipnull::Bool = false) =
mapreduce($f, $op, X; skipnull=skipnull)
end

# internal methods for Base.minimum and Base.maximum
for Op in (:(Base.MinFun), :(Base.MaxFun))
for op in (@functorize(scalarmin), @functorize(scalarmax))
@eval begin
function Base._mapreduce{T}(::Base.IdFun, ::$Op,
function Base._mapreduce{T}(::typeof(@functorize(identity)), ::$(typeof(op)),
X::NullableArray{T}, missingdata)
missingdata && return Nullable{T}()
Nullable(Base._mapreduce(Base.IdFun(), $Op(), X.values))
Nullable(Base._mapreduce(@functorize(identity), $op, X.values))
end
end
end

function Base.mapreduce_impl{T}(f, op::Base.MinFun, X::NullableArray{T},
function Base.mapreduce_impl{T}(f, op::typeof(@functorize(scalarmin)), X::NullableArray{T},
first::Int, last::Int)
i = first
v = f(X[i])
Expand All @@ -183,7 +193,7 @@ function Base.mapreduce_impl{T}(f, op::Base.MinFun, X::NullableArray{T},
return v
end

function Base.mapreduce_impl{T}(f, op::Base.MaxFun, X::NullableArray{T},
function Base.mapreduce_impl{T}(f, op::typeof(@functorize(scalarmax)), X::NullableArray{T},
first::Int, last::Int)
i = first
v = f(X[i])
Expand Down