From a9f21326062b0b455aebd2459e8406b03a048a26 Mon Sep 17 00:00:00 2001 From: Matthijs Hofstra Date: Sun, 12 May 2013 21:22:20 +0200 Subject: [PATCH 1/3] Adds atomic_load, atomic_load_acq, atomic_store, and atomic_store_rel intrinsics. The default versions (atomic_load and atomic_store) are sequentially consistent. The atomic_load_acq intrinsic acquires as described in [1]. The atomic_store_rel intrinsic releases as described in [1]. [1]: http://llvm.org/docs/Atomics.html --- src/libcore/unstable/intrinsics.rs | 10 ++++++++++ src/librustc/lib/llvm.rs | 22 +++++++++++++++++---- src/librustc/middle/trans/build.rs | 26 +++++++++++++++++++++++++ src/librustc/middle/trans/foreign.rs | 24 +++++++++++++++++++++++ src/librustc/middle/trans/type_use.rs | 4 +++- src/librustc/middle/typeck/check/mod.rs | 19 ++++++++++++++++++ src/rustllvm/RustWrapper.cpp | 22 +++++++++++++++++++++ src/rustllvm/rustllvm.def.in | 2 ++ src/test/run-pass/intrinsic-atomics.rs | 15 ++++++++++++++ 9 files changed, 139 insertions(+), 5 deletions(-) diff --git a/src/libcore/unstable/intrinsics.rs b/src/libcore/unstable/intrinsics.rs index a820c5d15a8db..04ffd169e98a3 100644 --- a/src/libcore/unstable/intrinsics.rs +++ b/src/libcore/unstable/intrinsics.rs @@ -19,6 +19,16 @@ pub extern "rust-intrinsic" { pub fn atomic_cxchg(dst: &mut int, old: int, src: int) -> int; pub fn atomic_cxchg_acq(dst: &mut int, old: int, src: int) -> int; pub fn atomic_cxchg_rel(dst: &mut int, old: int, src: int) -> int; + + #[cfg(not(stage0))] + pub fn atomic_load(src: &int) -> int; + #[cfg(not(stage0))] + pub fn atomic_load_acq(src: &int) -> int; + + #[cfg(not(stage0))] + pub fn atomic_store(dst: &mut int, val: int); + #[cfg(not(stage0))] + pub fn atomic_store_rel(dst: &mut int, val: int); pub fn atomic_xchg(dst: &mut int, src: int) -> int; pub fn atomic_xchg_acq(dst: &mut int, src: int) -> int; diff --git a/src/librustc/lib/llvm.rs b/src/librustc/lib/llvm.rs index 36d5a8e3cfe9e..dfd9df3bb5c15 100644 --- a/src/librustc/lib/llvm.rs +++ b/src/librustc/lib/llvm.rs @@ -1339,13 +1339,16 @@ pub mod llvm { PointerVal: ValueRef) -> ValueRef; #[fast_ffi] pub unsafe fn LLVMBuildLoad(B: BuilderRef, - PointerVal: ValueRef, - Name: *c_char) - -> ValueRef; + PointerVal: ValueRef, + Name: *c_char) + -> ValueRef; + #[fast_ffi] pub unsafe fn LLVMBuildStore(B: BuilderRef, Val: ValueRef, - Ptr: ValueRef) -> ValueRef; + Ptr: ValueRef) + -> ValueRef; + #[fast_ffi] pub unsafe fn LLVMBuildGEP(B: BuilderRef, Pointer: ValueRef, @@ -1561,6 +1564,17 @@ pub mod llvm { Name: *c_char) -> ValueRef; /* Atomic Operations */ + pub unsafe fn LLVMBuildAtomicLoad(B: BuilderRef, + PointerVal: ValueRef, + Order: AtomicOrdering) + -> ValueRef; + + pub unsafe fn LLVMBuildAtomicStore(B: BuilderRef, + Val: ValueRef, + Ptr: ValueRef, + Order: AtomicOrdering) + -> ValueRef; + pub unsafe fn LLVMBuildAtomicCmpXchg(B: BuilderRef, LHS: ValueRef, CMP: ValueRef, diff --git a/src/librustc/middle/trans/build.rs b/src/librustc/middle/trans/build.rs index 29fd90edce898..6fb2250871478 100644 --- a/src/librustc/middle/trans/build.rs +++ b/src/librustc/middle/trans/build.rs @@ -537,6 +537,21 @@ pub fn Load(cx: block, PointerVal: ValueRef) -> ValueRef { } } +pub fn AtomicLoad(cx: block, PointerVal: ValueRef, order: AtomicOrdering) -> ValueRef { + unsafe { + let ccx = cx.fcx.ccx; + if cx.unreachable { + let ty = val_ty(PointerVal); + let eltty = if llvm::LLVMGetTypeKind(ty) == lib::llvm::Array { + llvm::LLVMGetElementType(ty) } else { ccx.int_type }; + return llvm::LLVMGetUndef(eltty); + } + count_insn(cx, "load.atomic"); + return llvm::LLVMBuildAtomicLoad(B(cx), PointerVal, order); + } +} + + pub fn LoadRangeAssert(cx: block, PointerVal: ValueRef, lo: c_ulonglong, hi: c_ulonglong, signed: lib::llvm::Bool) -> ValueRef { let value = Load(cx, PointerVal); @@ -567,6 +582,17 @@ pub fn Store(cx: block, Val: ValueRef, Ptr: ValueRef) { } } +pub fn AtomicStore(cx: block, Val: ValueRef, Ptr: ValueRef, order: AtomicOrdering) { + unsafe { + if cx.unreachable { return; } + debug!("Store %s -> %s", + val_str(cx.ccx().tn, Val), + val_str(cx.ccx().tn, Ptr)); + count_insn(cx, "store.atomic"); + llvm::LLVMBuildAtomicStore(B(cx), Val, Ptr, order); + } +} + pub fn GEP(cx: block, Pointer: ValueRef, Indices: &[ValueRef]) -> ValueRef { unsafe { if cx.unreachable { return llvm::LLVMGetUndef(T_ptr(T_nil())); } diff --git a/src/librustc/middle/trans/foreign.rs b/src/librustc/middle/trans/foreign.rs index 26654cf31f861..30db63e9c19da 100644 --- a/src/librustc/middle/trans/foreign.rs +++ b/src/librustc/middle/trans/foreign.rs @@ -592,6 +592,30 @@ pub fn trans_intrinsic(ccx: @CrateContext, Release); Store(bcx, old, fcx.llretptr.get()); } + ~"atomic_load" => { + let old = AtomicLoad(bcx, + get_param(decl, first_real_arg), + SequentiallyConsistent); + Store(bcx, old, fcx.llretptr.get()); + } + ~"atomic_load_acq" => { + let old = AtomicLoad(bcx, + get_param(decl, first_real_arg), + Acquire); + Store(bcx, old, fcx.llretptr.get()); + } + ~"atomic_store" => { + AtomicStore(bcx, + get_param(decl, first_real_arg + 1u), + get_param(decl, first_real_arg), + SequentiallyConsistent); + } + ~"atomic_store_rel" => { + AtomicStore(bcx, + get_param(decl, first_real_arg + 1u), + get_param(decl, first_real_arg), + Release); + } ~"atomic_xchg" => { let old = AtomicRMW(bcx, Xchg, get_param(decl, first_real_arg), diff --git a/src/librustc/middle/trans/type_use.rs b/src/librustc/middle/trans/type_use.rs index 3c2738d3ae80e..6a271f979caed 100644 --- a/src/librustc/middle/trans/type_use.rs +++ b/src/librustc/middle/trans/type_use.rs @@ -124,7 +124,9 @@ pub fn type_uses_for(ccx: @CrateContext, fn_id: def_id, n_tps: uint) ~"get_tydesc" | ~"needs_drop" => use_tydesc, ~"atomic_cxchg" | ~"atomic_cxchg_acq"| - ~"atomic_cxchg_rel"| ~"atomic_xchg" | + ~"atomic_cxchg_rel"| ~"atomic_load" | + ~"atomic_load_acq" | ~"atomic_store" | + ~"atomic_store_rel"| ~"atomic_xchg" | ~"atomic_xadd" | ~"atomic_xsub" | ~"atomic_xchg_acq" | ~"atomic_xadd_acq" | ~"atomic_xsub_acq" | ~"atomic_xchg_rel" | diff --git a/src/librustc/middle/typeck/check/mod.rs b/src/librustc/middle/typeck/check/mod.rs index 7c79693a8b2eb..7f73ed94c031c 100644 --- a/src/librustc/middle/typeck/check/mod.rs +++ b/src/librustc/middle/typeck/check/mod.rs @@ -3486,6 +3486,25 @@ pub fn check_intrinsic_type(ccx: @mut CrateCtxt, it: @ast::foreign_item) { ], ty::mk_int()) } + ~"atomic_load" | ~"atomic_load_acq" => { + (0, + ~[ + arg(ty::mk_imm_rptr(tcx, + ty::re_bound(ty::br_anon(0)), + ty::mk_int())) + ], + ty::mk_int()) + } + ~"atomic_store" | ~"atomic_store_rel" => { + (0, + ~[ + arg(ty::mk_mut_rptr(tcx, + ty::re_bound(ty::br_anon(0)), + ty::mk_int())), + arg(ty::mk_int()) + ], + ty::mk_nil()) + } ~"atomic_xchg" | ~"atomic_xadd" | ~"atomic_xsub" | ~"atomic_xchg_acq" | ~"atomic_xadd_acq" | ~"atomic_xsub_acq" | ~"atomic_xchg_rel" | ~"atomic_xadd_rel" | ~"atomic_xsub_rel" => { diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index 04e616de22334..8c081858d602a 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -545,6 +545,28 @@ extern "C" LLVMTypeRef LLVMMetadataType(void) { return LLVMMetadataTypeInContext(LLVMGetGlobalContext()); } +extern "C" LLVMValueRef LLVMBuildAtomicLoad(LLVMBuilderRef B, + LLVMValueRef source, + const char* Name, + AtomicOrdering order) { + LoadInst* li = new LoadInst(unwrap(source),0); + li->setVolatile(true); + li->setAtomic(order); + li->setAlignment(sizeof(intptr_t)); + return wrap(unwrap(B)->Insert(li)); +} + +extern "C" LLVMValueRef LLVMBuildAtomicStore(LLVMBuilderRef B, + LLVMValueRef val, + LLVMValueRef target, + AtomicOrdering order) { + StoreInst* si = new StoreInst(unwrap(val),unwrap(target)); + si->setVolatile(true); + si->setAtomic(order); + si->setAlignment(sizeof(intptr_t)); + return wrap(unwrap(B)->Insert(si)); +} + extern "C" LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef target, LLVMValueRef old, diff --git a/src/rustllvm/rustllvm.def.in b/src/rustllvm/rustllvm.def.in index 73bf1af90cd34..dd5dc7102d1e9 100644 --- a/src/rustllvm/rustllvm.def.in +++ b/src/rustllvm/rustllvm.def.in @@ -84,6 +84,8 @@ LLVMArrayType LLVMBasicBlockAsValue LLVMBlockAddress LLVMBuildAShr +LLVMBuildAtomicLoad +LLVMBuildAtomicStore LLVMBuildAtomicCmpXchg LLVMBuildAtomicRMW LLVMBuildAdd diff --git a/src/test/run-pass/intrinsic-atomics.rs b/src/test/run-pass/intrinsic-atomics.rs index 42d4f5e4d2078..d4701f74488c8 100644 --- a/src/test/run-pass/intrinsic-atomics.rs +++ b/src/test/run-pass/intrinsic-atomics.rs @@ -15,6 +15,12 @@ mod rusti { pub fn atomic_cxchg_acq(dst: &mut int, old: int, src: int) -> int; pub fn atomic_cxchg_rel(dst: &mut int, old: int, src: int) -> int; + pub fn atomic_load(src: &int) -> int; + pub fn atomic_load_acq(src: &int) -> int; + + pub fn atomic_store(dst: &mut int, val: int); + pub fn atomic_store_rel(dst: &mut int, val: int); + pub fn atomic_xchg(dst: &mut int, src: int) -> int; pub fn atomic_xchg_acq(dst: &mut int, src: int) -> int; pub fn atomic_xchg_rel(dst: &mut int, src: int) -> int; @@ -33,6 +39,15 @@ pub fn main() { unsafe { let mut x = ~1; + assert!(rusti::atomic_load(x) == 1); + *x = 5; + assert!(rusti::atomic_load_acq(x) == 5); + + rusti::atomic_store(x,3); + assert!(*x == 3); + rusti::atomic_store_rel(x,1); + assert!(*x == 1); + assert!(rusti::atomic_cxchg(x, 1, 2) == 1); assert!(*x == 2); From fa1a172f9c8b5071a7bc9d818bfa6ae26401ec5b Mon Sep 17 00:00:00 2001 From: Matthijs Hofstra Date: Mon, 13 May 2013 00:12:14 +0200 Subject: [PATCH 2/3] Removed unnecessary check from build.rs --- src/librustc/middle/trans/build.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/librustc/middle/trans/build.rs b/src/librustc/middle/trans/build.rs index 6fb2250871478..b2af91887ecbf 100644 --- a/src/librustc/middle/trans/build.rs +++ b/src/librustc/middle/trans/build.rs @@ -541,10 +541,7 @@ pub fn AtomicLoad(cx: block, PointerVal: ValueRef, order: AtomicOrdering) -> Val unsafe { let ccx = cx.fcx.ccx; if cx.unreachable { - let ty = val_ty(PointerVal); - let eltty = if llvm::LLVMGetTypeKind(ty) == lib::llvm::Array { - llvm::LLVMGetElementType(ty) } else { ccx.int_type }; - return llvm::LLVMGetUndef(eltty); + return llvm::LLVMGetUndef(ccx.int_type); } count_insn(cx, "load.atomic"); return llvm::LLVMBuildAtomicLoad(B(cx), PointerVal, order); From 852af346d346e0119af8336c88ee1b113346ac7c Mon Sep 17 00:00:00 2001 From: Matthijs Hofstra Date: Mon, 13 May 2013 13:33:34 +0200 Subject: [PATCH 3/3] Tidy --- src/libcore/unstable/intrinsics.rs | 4 ++-- src/librustc/lib/llvm.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libcore/unstable/intrinsics.rs b/src/libcore/unstable/intrinsics.rs index 04ffd169e98a3..b8c0c4e4a9292 100644 --- a/src/libcore/unstable/intrinsics.rs +++ b/src/libcore/unstable/intrinsics.rs @@ -19,12 +19,12 @@ pub extern "rust-intrinsic" { pub fn atomic_cxchg(dst: &mut int, old: int, src: int) -> int; pub fn atomic_cxchg_acq(dst: &mut int, old: int, src: int) -> int; pub fn atomic_cxchg_rel(dst: &mut int, old: int, src: int) -> int; - + #[cfg(not(stage0))] pub fn atomic_load(src: &int) -> int; #[cfg(not(stage0))] pub fn atomic_load_acq(src: &int) -> int; - + #[cfg(not(stage0))] pub fn atomic_store(dst: &mut int, val: int); #[cfg(not(stage0))] diff --git a/src/librustc/lib/llvm.rs b/src/librustc/lib/llvm.rs index dfd9df3bb5c15..4c61c42a33932 100644 --- a/src/librustc/lib/llvm.rs +++ b/src/librustc/lib/llvm.rs @@ -1574,7 +1574,7 @@ pub mod llvm { Ptr: ValueRef, Order: AtomicOrdering) -> ValueRef; - + pub unsafe fn LLVMBuildAtomicCmpXchg(B: BuilderRef, LHS: ValueRef, CMP: ValueRef,