diff --git a/src/core/instructions.jl b/src/core/instructions.jl index 1937037f..6dd7af06 100644 --- a/src/core/instructions.jl +++ b/src/core/instructions.jl @@ -129,6 +129,18 @@ for op in opcodes end +## atomics + +export ordering, ordering! + +# Ordering getter/setter are supported only for a subset of instructions +# https://github.com/llvm/llvm-project/blob/llvmorg-14.0.3/llvm/lib/IR/Core.cpp#L3779-L3798 + +ordering(val::Union{LoadInst,StoreInst,AtomicRMWInst}) = API.LLVMGetOrdering(val) +ordering!(val::Union{LoadInst,StoreInst}, ord::API.LLVMAtomicOrdering) = + API.LLVMSetOrdering(val, ord) + + ## call sites and invocations # TODO: add this to the actual type hierarchy diff --git a/src/interop.jl b/src/interop.jl index ea6ce21a..1bd0b66f 100644 --- a/src/interop.jl +++ b/src/interop.jl @@ -10,6 +10,7 @@ include("interop/base.jl") include("interop/asmcall.jl") include("interop/passes.jl") include("interop/pointer.jl") +include("interop/atomics.jl") include("interop/utils.jl") include("interop/intrinsics.jl") diff --git a/src/interop/atomics.jl b/src/interop/atomics.jl new file mode 100644 index 00000000..e8ce1523 --- /dev/null +++ b/src/interop/atomics.jl @@ -0,0 +1,538 @@ +const MEMORY_ORDERING_EXPLANATION = """ +specified as a symbol (e.g., `:sequentially_consistent`) or a `Val` of a symbol (e.g., +`Val(:sequentially_consistent)`) +""" + +""" + LLVM.Interop.atomic_pointerref(pointer::LLVMPtr{T}, ordering) -> value::T + +Load a `value` from `pointer` with the given memory `ordering` atomically. + +`ordering` is a Julia atomic ordering $MEMORY_ORDERING_EXPLANATION. + +See also: `getproperty`, `getfield` +""" +atomic_pointerref + +""" + LLVM.Interop.atomic_pointerset(pointer::LLVMPtr{T}, x::T, ordering) -> pointer + +Store a value `x` in `pointer` with the given memory `ordering` atomically. + +`ordering` is a Julia atomic ordering $MEMORY_ORDERING_EXPLANATION. + +See also: `setproperty!`, `setfield!` +""" +atomic_pointerset + +""" + LLVM.Interop.atomic_pointermodify( + pointer::LLVMPtr{T}, + op, + x::T, + ordering, + ) -> (old => new)::Pair{T,T} + +Replace an `old` value stored at `pointer` with a `new` value comped as `new = op(old, x)` +with the given memory `ordering` atomically. Return a pair `old => new`. + +`ordering` is a Julia atomic ordering $MEMORY_ORDERING_EXPLANATION. + +See also: `modifyproperty!`, `modifyfield!` +""" +atomic_pointermodify + +""" + LLVM.Interop.atomic_pointerswap(pointer::LLVMPtr{T}, op, new::T, ordering) -> old::T + +Replace an `old` value stored at `pointer` with a `new` value with the given memory +`ordering` atomically. Return the `old` value. + +`ordering` is a Julia atomic ordering $MEMORY_ORDERING_EXPLANATION. + +See also: `modifyproperty!`, `modifyfield!` +""" +atomic_pointerswap + +""" + LLVM.Interop.atomic_pointerreplace( + pointer::LLVMPtr{T}, + expected::T, + desired::T, + success_ordering, + fail_ordering, + ) -> (; old::T, success::Bool) + +Try to replace the value of `pointer` from an `expected` value to a `desired` value +atomically with the ordering `success_ordering`. The property `old` of the returned value +is the value stored in the `pointer`. The property `success` of the returned value +indicates if the replacement was successful. The ordering `fail_ordering` specifies the +ordering used for loading the `old` value. + +`success_ordering` and `fail_ordering` are Julia atomic orderings +$MEMORY_ORDERING_EXPLANATION. + +See also: `replaceproperty!`, `replacefield!` +""" +atomic_pointerreplace + +const _llvm_from_julia_ordering = ( + not_atomic = LLVM.API.LLVMAtomicOrderingNotAtomic, + unordered = LLVM.API.LLVMAtomicOrderingUnordered, + monotonic = LLVM.API.LLVMAtomicOrderingMonotonic, + acquire = LLVM.API.LLVMAtomicOrderingAcquire, + release = LLVM.API.LLVMAtomicOrderingRelease, + acquire_release = LLVM.API.LLVMAtomicOrderingAcquireRelease, + sequentially_consistent = LLVM.API.LLVMAtomicOrderingSequentiallyConsistent, +) + +_julia_ordering(p) = + Union{map(x -> p(x) ? Val{x} : Union{}, keys(_llvm_from_julia_ordering))...} + +const AllOrdering = _julia_ordering(_ -> true) +const AtomicOrdering = _julia_ordering(!=(:not_atomic)) + +const LLVMOrderingVal = Union{map(x -> Val{x}, values(_llvm_from_julia_ordering))...} + +is_stronger_than_monotonic(order::Symbol) = + !(order === :monotonic || order === :unordered || order === :not_atomic) + +for (julia, llvm) in pairs(_llvm_from_julia_ordering) + @eval llvm_from_julia_ordering(::Val{$(QuoteNode(julia))}) = Val{$llvm}() +end + +""" + @dynamic_order(order) do order + ... use order ... + end + +It is expanded to an expression similar to: + + if order === :not_atomic + let order = Val(:not_atomic) + ... use order ... + end + elseif order === :unordered + let order = Val(:unordered) + ... use order ... + end + elseif ... + ... + else + throw(ConcurrencyViolationError(...)) + end + +This is used for helping the compiler to optimize expressions such as +`atomic_pointerref(ptr, :monotonic)` and also to avoid abstract run-time dispatch. +""" +macro dynamic_order(thunk, order) + @assert Meta.isexpr(thunk, :->, 2) && Meta.isexpr(thunk.args[1], :tuple, 1) + ordervar = esc(thunk.args[1].args[1]) + body = esc(thunk.args[2]) + expr = foldr( + keys(_llvm_from_julia_ordering), + init = :(throw(ConcurrencyViolationError("invalid atomic ordering: ", order))), + ) do key, r + quote + if order === $(QuoteNode(key)) + let $ordervar = Val{$(QuoteNode(key))}() + $body + end + else + $r + end + end + end + quote + order = $(esc(order)) + $expr + end +end + +_valueof(::Val{x}) where {x} = x + +@inline function atomic_pointerref(pointer, order::Symbol) + @dynamic_order(order) do order + atomic_pointerref(pointer, order) + end +end + +@inline function atomic_pointerset(pointer, x, order::Symbol) + @dynamic_order(order) do order + atomic_pointerset(pointer, x, order) + end +end + +@generated function atomic_pointerref(ptr::LLVMPtr{T,A}, order::AllOrdering) where {T,A} + sizeof(T) == 0 && return T.instance + llvm_order = _valueof(llvm_from_julia_ordering(order())) + Context() do ctx + eltyp = convert(LLVMType, T; ctx) + + T_ptr = convert(LLVMType, ptr; ctx) + + T_typed_ptr = LLVM.PointerType(eltyp, A) + + # create a function + param_types = [T_ptr] + llvm_f, _ = create_function(eltyp, param_types) + + # generate IR + Builder(ctx) do builder + entry = BasicBlock(llvm_f, "entry"; ctx) + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + ld = load!(builder, typed_ptr) + ordering!(ld, llvm_order) + + if A != 0 + metadata(ld)[LLVM.MD_tbaa] = tbaa_addrspace(A; ctx) + end + alignment!(ld, sizeof(T)) + + ret!(builder, ld) + end + + call_function(llvm_f, T, Tuple{LLVMPtr{T,A}}, :ptr) + end +end + +@generated function atomic_pointerset( + ptr::LLVMPtr{T,A}, + x::T, + order::AllOrdering, +) where {T,A} + if sizeof(T) == 0 + # Mimicking what `Core.Intrinsics.atomic_pointerset` generates. + # See: https://github.com/JuliaLang/julia/blob/v1.7.2/src/cgutils.cpp#L1570-L1572 + is_stronger_than_monotonic(order) || return :ptr + return quote + Core.Intrinsics.fence($(QuoteNode(order))) + ptr + end + end + llvm_order = _valueof(llvm_from_julia_ordering(order())) + Context() do ctx + eltyp = convert(LLVMType, T; ctx) + T_ptr = convert(LLVMType, ptr; ctx) + T_typed_ptr = LLVM.PointerType(eltyp, A) + + # create a function + param_types = [T_ptr, eltyp] + llvm_f, _ = create_function(LLVM.VoidType(ctx), param_types) + + # generate IR + Builder(ctx) do builder + entry = BasicBlock(llvm_f, "entry"; ctx) + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + val = parameters(llvm_f)[2] + st = store!(builder, val, typed_ptr) + ordering!(st, llvm_order) + + if A != 0 + metadata(st)[LLVM.MD_tbaa] = tbaa_addrspace(A; ctx) + end + alignment!(st, sizeof(T)) + + ret!(builder) + end + + call = call_function(llvm_f, Cvoid, Tuple{LLVMPtr{T,A},T}, :ptr, :x) + quote + $call + ptr + end + end +end + +right(_, r) = r + +const binoptable = [ + (:xchg, right, LLVM.API.LLVMAtomicRMWBinOpXchg), + (:add, +, LLVM.API.LLVMAtomicRMWBinOpAdd), + (:sub, -, LLVM.API.LLVMAtomicRMWBinOpSub), + (:and, &, LLVM.API.LLVMAtomicRMWBinOpAnd), + (:or, |, LLVM.API.LLVMAtomicRMWBinOpOr), + (:xor, xor, LLVM.API.LLVMAtomicRMWBinOpXor), + (:max, max, LLVM.API.LLVMAtomicRMWBinOpMax), + (:min, min, LLVM.API.LLVMAtomicRMWBinOpMin), + (:umax, max, LLVM.API.LLVMAtomicRMWBinOpUMax), + (:umin, min, LLVM.API.LLVMAtomicRMWBinOpUMin), + (:fadd, +, LLVM.API.LLVMAtomicRMWBinOpFAdd), + (:fsub, -, LLVM.API.LLVMAtomicRMWBinOpFSub), +] + +const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} + +@generated function llvm_atomic_op( + binop::AtomicRMWBinOpVal, + ptr::LLVMPtr{T,A}, + val::T, + order::LLVMOrderingVal, +) where {T,A} + Context() do ctx + T_val = convert(LLVMType, T; ctx) + T_ptr = convert(LLVMType, ptr; ctx) + + T_typed_ptr = LLVM.PointerType(T_val, A) + + llvm_f, _ = create_function(T_val, [T_ptr, T_val]) + + Builder(ctx) do builder + entry = BasicBlock(llvm_f, "entry"; ctx) + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + + single_threaded = false + rv = atomic_rmw!( + builder, + _valueof(binop()), + typed_ptr, + parameters(llvm_f)[2], + _valueof(order()), + single_threaded, + ) + + ret!(builder, rv) + end + + call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val) + end +end + +@inline function atomic_pointermodify(pointer, op::OP, x, order::Symbol) where {OP} + @dynamic_order(order) do order + atomic_pointermodify(pointer, op, x, order) + end +end + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + op, + x::T, + ::Val{:not_atomic}, +) where {T} + old = atomic_pointerref(ptr, Val(:not_atomic)) + new = op(old, x) + atomic_pointerset(ptr, new, Val(:not_atomic)) + return old => new +end + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + ::typeof(right), + x::T, + order::AtomicOrdering, +) where {T} + old = llvm_atomic_op( + Val(LLVM.API.LLVMAtomicRMWBinOpXchg), + ptr, + x, + llvm_from_julia_ordering(order), + ) + return old => x +end + +const atomictypes = Any[ + Int8, + Int16, + Int32, + Int64, + Int128, + UInt8, + UInt16, + UInt32, + UInt64, + UInt128, + Float16, + Float32, + Float64, +] + +for (opname, op, llvmop) in binoptable + opname === :xchg && continue + types = if opname in (:min, :max) + filter(t -> t <: Signed, atomictypes) + elseif opname in (:umin, :umax) + filter(t -> t <: Unsigned, atomictypes) + elseif opname in (:fadd, :fsub) + filter(t -> t <: AbstractFloat, atomictypes) + else + filter(t -> t <: Integer, atomictypes) + end + for T in types + @eval @inline function atomic_pointermodify( + ptr::LLVMPtr{$T}, + ::$(typeof(op)), + x::$T, + order::AtomicOrdering, + ) + old = llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) + return old => $op(old, x) + end + end +end + +@inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) +@inline atomic_pointerswap(pointer, new, order) = + first(atomic_pointermodify(pointer, right, new, order)) + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + op, + x::T, + order::AllOrdering, +) where {T} + # Should `fail_order` be stronger? Ref: https://github.com/JuliaLang/julia/issues/45256 + fail_order = Val(:monotonic) + old = atomic_pointerref(ptr, fail_order) + while true + new = op(old, x) + (old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order) + success && return old => new + end +end + +@generated function llvm_atomic_cas( + ptr::LLVMPtr{T,A}, + cmp::T, + val::T, + success_order::LLVMOrderingVal, + fail_order::LLVMOrderingVal, +) where {T,A} + llvm_success = _valueof(success_order()) + llvm_fail = _valueof(fail_order()) + Context() do ctx + T_val = convert(LLVMType, T; ctx) + T_pointee = T_val + if T_val isa LLVM.FloatingPointType + T_pointee = LLVM.IntType(sizeof(T) * 8; ctx) + end + T_ptr = convert(LLVMType, ptr; ctx) + T_success = convert(LLVMType, Ptr{Int8}; ctx) + + T_typed_ptr = LLVM.PointerType(T_pointee, A) + T_ok_ptr = LLVM.PointerType(convert(LLVMType, Int8; ctx)) + + llvm_f, _ = create_function(T_val, [T_ptr, T_val, T_val, T_success]) + + Builder(ctx) do builder + entry = BasicBlock(llvm_f, "entry"; ctx) + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + ok_ptr = inttoptr!(builder, parameters(llvm_f)[4], T_ok_ptr) + + cmp_int = parameters(llvm_f)[2] + if T_val isa LLVM.FloatingPointType + cmp_int = bitcast!(builder, cmp_int, T_pointee) + end + + val_int = parameters(llvm_f)[3] + if T_val isa LLVM.FloatingPointType + val_int = bitcast!(builder, val_int, T_pointee) + end + + single_threaded = false + res = atomic_cmpxchg!( + builder, + typed_ptr, + cmp_int, + val_int, + llvm_success, + llvm_fail, + single_threaded, + ) + + rv = extract_value!(builder, res, 0) + ok = extract_value!(builder, res, 1) + ok = zext!(builder, ok, LLVM.Int8Type(ctx)) + store!(builder, ok, ok_ptr) + + if T_val isa LLVM.FloatingPointType + rv = bitcast!(builder, rv, T_val) + end + + ret!(builder, rv) + end + + expr = call_function( + llvm_f, + T, + Tuple{LLVMPtr{T,A},T,T,Ptr{Int8}}, + :ptr, + :cmp, + :val, + :success_ptr, + ) + quote + success = Ref{Int8}() + old = GC.@preserve success begin + success_ptr = Ptr{Int8}(pointer_from_objref(success)) + $expr + end + (; old, success = success[] != zero(Int8)) + end + end +end + +@inline function atomic_pointerreplace( + pointer, + expected, + desired, + success_order::Symbol, + fail_order::Symbol, +) + # This avoids abstract dispatch at run-time but probably too much codegen? + #= + @dynamic_order(success_order) do success_order + @dynamic_order(fail_order) do fail_order + atomic_pointerreplace(pointer, expected, desired, success_order, fail_order) + end + end + =# + + # This avoids excessive codegen while hopefully imposes no cost when const-prop works: + so = @dynamic_order(success_order) do success_order + success_order + end + fo = @dynamic_order(fail_order) do fail_order + fail_order + end + return atomic_pointerreplace(pointer, expected, desired, so, fo) +end + +@inline function atomic_pointerreplace( + ptr::LLVMPtr{T}, + expected::T, + desired::T, + ::Val{:not_atomic}, + ::Val{:not_atomic}, +) where {T} + old = atomic_pointerref(ptr, Val(:not_atomic)) + if old === expected + atomic_pointerset(ptr, desired, Val(:not_atomic)) + success = true + else + success = false + end + return (; old, success) +end + +@inline atomic_pointerreplace( + ptr::LLVMPtr{T}, + expected::T, + desired::T, + success_order::_julia_ordering(∉((:not_atomic, :unordered))), + fail_order::_julia_ordering(∉((:not_atomic, :unordered, :release, :acquire_release))), +) where {T} = llvm_atomic_cas( + ptr, + expected, + desired, + llvm_from_julia_ordering(success_order), + llvm_from_julia_ordering(fail_order), +) diff --git a/test/atomics.jl b/test/atomics.jl new file mode 100644 index 00000000..5e88ac06 --- /dev/null +++ b/test/atomics.jl @@ -0,0 +1,220 @@ +using InteractiveUtils +using LLVM.Interop +using Test + +@testset "atomics" begin + +coreptr(r::Base.RefValue) = reinterpret(Ptr{eltype(r)}, pointer_from_objref(r)) +llvmptr(r::Base.RefValue) = reinterpret(Core.LLVMPtr{eltype(r),0}, pointer_from_objref(r)) + +# Testing that ordering can be const-prop'ed +atomic_pointerref_monotonic(ptr) = Interop.atomic_pointerref(ptr, :monotonic) +atomic_pointerset_monotonic(ptr, x) = Interop.atomic_pointerset(ptr, x, :monotonic) +atomic_pointermodify_monotonic(ptr, op::OP, x) where {OP} = + Interop.atomic_pointermodify(ptr, op, x, :monotonic) +atomic_pointerswap_monotonic(ptr, x) = Interop.atomic_pointerswap(ptr, x, :monotonic) +atomic_pointerreplace_monotonic(ptr, expected, desired) = + Interop.atomic_pointerreplace(ptr, expected, desired, :monotonic, :monotonic) + +muladd1(x, y) = muladd(x, y, one(x)) + +MONOTONIC = :monotonic + +@testset for T in + [Int8, Int16, Int32, Int64, UInt8, UInt16, UInt32, UInt64, Float32, Float64] + r1 = Ref(one(T)) + r2 = Ref(one(T)) + GC.@preserve r1 r2 begin + p1 = llvmptr(r1) + sp1 = coreptr(r1) + sp2 = coreptr(r2) + + @test atomic_pointerref_monotonic(p1) === r1[] + @test Interop.atomic_pointerref(p1, MONOTONIC) === r1[] + @test Interop.atomic_pointerref(p1, Val(:monotonic)) === r1[] + if VERSION ≥ v"1.7" + @test Interop.atomic_pointerref(p1, Val(:monotonic)) === + Core.Intrinsics.atomic_pointerref(sp1, :monotonic) + end + + types = typeof((p1, Val(:sequentially_consistent))) + ir = sprint(io -> code_llvm(io, Interop.atomic_pointerref, types)) + @test occursin(r"load atomic .* seq_cst"m, ir) + + ir = sprint(io -> code_llvm(io, atomic_pointerref_monotonic, Tuple{typeof(p1)})) + @test occursin(r"load atomic .* monotonic"m, ir) + @test !occursin(r"load atomic .* seq_cst"m, ir) + + val = r1[] + one(r1[]) + @test begin + Interop.atomic_pointerset(p1, val, MONOTONIC) + r1[] + end === val + + val = r1[] + one(r1[]) + @test begin + Interop.atomic_pointerset(p1, val, Val(:monotonic)) + r1[] + end === val + + if VERSION ≥ v"1.7" + val = r1[] + one(r1[]) + r2[] = r1[] + @test begin + Interop.atomic_pointerset(p1, val, Val(:monotonic)) + r1[] + end === begin + Core.Intrinsics.atomic_pointerset(sp2, val, :monotonic) + r2[] + end + end + + types = typeof((p1, val, Val(:sequentially_consistent))) + ir = sprint(io -> code_llvm(io, Interop.atomic_pointerset, types)) + @test occursin(r"store atomic .* seq_cst"m, ir) + + ir = sprint(io -> code_llvm(io, atomic_pointerset_monotonic, typeof((p1, val)))) + @test occursin(r"store atomic .* monotonic"m, ir) + @test !occursin(r"store atomic .* seq_cst"m, ir) + + ops = if T <: AbstractFloat + Any[Interop.right, +, -] + else + Any[op for (_, op, _) in Interop.binoptable] + end + push!(ops, muladd1) + + old = r1[] + @testset for op in ops + r1[] = old + val = one(old) + @test begin + Interop.atomic_pointermodify(p1, op, val, MONOTONIC) + r1[] + end === op(old, val) + + r1[] = old + val = one(old) + @test begin + Interop.atomic_pointermodify(p1, op, val, Val(:monotonic)) + r1[] + end === op(old, val) + + if VERSION ≥ v"1.7" + r1[] = r2[] = old + val = one(old) + @test begin + Interop.atomic_pointermodify(p1, op, val, Val(:monotonic)) + r1[] + end === begin + Core.Intrinsics.atomic_pointermodify(sp2, op, val, :monotonic) + r2[] + end + end + + types = typeof((p1, op, val, Val(:sequentially_consistent))) + ir = sprint(io -> code_llvm(io, Interop.atomic_pointermodify, types)) + if op === muladd1 + @test occursin(r"cmpxchg .* seq_cst"m, ir) + else + @test occursin(r"atomicrmw .* seq_cst"m, ir) + end + + types = typeof((p1, op, val)) + ir = sprint(io -> code_llvm(io, atomic_pointermodify_monotonic, types)) + if op === muladd1 + @test occursin(r"cmpxchg .* monotonic"m, ir) + @test !occursin(r"cmpxchg .* seq_cst"m, ir) + else + @test occursin(r"atomicrmw .* monotonic"m, ir) + @test !occursin(r"atomicrmw .* seq_cst"m, ir) + end + end + + r1[] = old + val = one(old) + @test (atomic_pointerswap_monotonic(p1, val), r1[]) === (old, val) + + r1[] = old + val = one(old) + @test (Interop.atomic_pointerswap(p1, val, MONOTONIC), r1[]) === (old, val) + + r1[] = old + val = one(old) + @test (Interop.atomic_pointerswap(p1, val, Val(:monotonic)), r1[]) === (old, val) + + if VERSION ≥ v"1.7" + r1[] = r2[] = old + val = one(old) + @test (Interop.atomic_pointerswap(p1, val, Val(:monotonic)), r1[]) === + (Core.Intrinsics.atomic_pointerswap(sp2, val, :monotonic), r2[]) + end + + types = typeof((p1, val, Val(:sequentially_consistent))) + ir = sprint(io -> code_llvm(io, Interop.atomic_pointerswap, types)) + @test occursin(r"atomicrmw xchg .* seq_cst"m, ir) + + ir = sprint(io -> code_llvm(io, atomic_pointerswap_monotonic, typeof((p1, val)))) + @test occursin(r"atomicrmw xchg .* monotonic"m, ir) + @test !occursin(r"atomicrmw xchg .* seq_cst"m, ir) + + r1[] = old + val = old + one(old) + @test (atomic_pointerreplace_monotonic(p1, old, val), r1[]) === + ((; old, success = true), val) + @test (atomic_pointerreplace_monotonic(p1, old, val), r1[]) === + ((; old = val, success = false), val) + + r1[] = old + val = old + one(old) + @test (Interop.atomic_pointerreplace(p1, old, val, MONOTONIC, MONOTONIC), r1[]) === + ((; old, success = true), val) + @test (Interop.atomic_pointerreplace(p1, old, val, MONOTONIC, MONOTONIC), r1[]) === + ((; old = val, success = false), val) + + r1[] = old + val = old + one(old) + @test ( + Interop.atomic_pointerreplace(p1, old, val, Val(:monotonic), Val(:monotonic)), + r1[], + ) === ((; old, success = true), val) + @test ( + Interop.atomic_pointerreplace(p1, old, val, Val(:monotonic), Val(:monotonic)), + r1[], + ) === ((; old = val, success = false), val) + + types = typeof((p1, old, val, Val(:acquire_release), Val(:acquire))) + ir = sprint(io -> code_llvm(io, Interop.atomic_pointerreplace, types)) + @test occursin(r"cmpxchg .* acq_rel acquire"m, ir) + + types = typeof((p1, old, val)) + ir = sprint(io -> code_llvm(io, atomic_pointerreplace_monotonic, types)) + @test occursin(r"cmpxchg .* monotonic monotonic"m, ir) + @test !occursin(r"cmpxchg .* seq_cst seq_cst"m, ir) + end +end + +primitive type B64 64 end +B64(x) = reinterpret(B64, convert(Int64, x)) + +@testset "primitive type" begin + ref = Ref(B64(123)) + ord = Val(:monotonic) + GC.@preserve ref begin + ptr = llvmptr(ref) + @test Interop.atomic_pointerref(ptr, ord) === ref[] + @test begin + Interop.atomic_pointerset(ptr, B64(456), ord) + ref[] + end === B64(456) + @test Interop.atomic_pointerswap(ptr, B64(789), ord) === B64(456) + @test ref[] === B64(789) + @test Interop.atomic_pointerreplace(ptr, B64(789), B64(123), ord, ord) === + (old = B64(789), success = true) + @test ref[] === B64(123) + @test Interop.atomic_pointerreplace(ptr, B64(789), B64(123), ord, ord) === + (old = B64(123), success = false) + end +end + +end diff --git a/test/instructions.jl b/test/instructions.jl index 1edf7220..90eab982 100644 --- a/test/instructions.jl +++ b/test/instructions.jl @@ -149,6 +149,10 @@ LLVM.Module("SomeModule"; ctx) do mod alignment!(loadinst, 4) @test alignment(loadinst) == 4 + ordering!(loadinst, LLVM.API.LLVMAtomicOrderingSequentiallyConsistent) + @check_ir loadinst "load atomic i32, i32* %4 seq_cst" + @test ordering(loadinst) == LLVM.API.LLVMAtomicOrderingSequentiallyConsistent + storeinst = store!(builder, int1, ptr1) @check_ir storeinst "store i32 %0, i32* %4" diff --git a/test/runtests.jl b/test/runtests.jl index 494e98d9..4fd665a0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -79,5 +79,6 @@ include("Kaleidoscope.jl") include("examples.jl") include("interop.jl") +include("atomics.jl") end