Skip to content

Commit 7bfa20b

Browse files
borsbors
committed
Auto merge of #50648 - nox:volatile-store, r=eddyb
Fix volatile_store and nontemporal_store Fixes #50371.
2 parents 8f39dba + 3ebe867 commit 7bfa20b

File tree

9 files changed

+75
-81
lines changed

9 files changed

+75
-81
lines changed

‎src/Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/librustc_trans/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ crate-type = ["dylib"]
1010
test = false
1111

1212
[dependencies]
13+
bitflags = "1.0.1"
1314
cc = "1.0.1"
1415
flate2 = "1.0"
1516
jobserver = "0.1.5"

‎src/librustc_trans/abi.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
use llvm::{self, ValueRef, AttributePlace};
1212
use base;
13-
use builder::Builder;
13+
use builder::{Builder, MemFlags};
1414
use common::{ty_fn_sig, C_usize};
1515
use context::CodegenCx;
1616
use mir::place::PlaceRef;
@@ -220,7 +220,8 @@ impl<'a, 'tcx> ArgTypeExt<'a, 'tcx> for ArgType<'tcx, Ty<'tcx>> {
220220
bx.pointercast(dst.llval, Type::i8p(cx)),
221221
bx.pointercast(llscratch, Type::i8p(cx)),
222222
C_usize(cx, self.layout.size.bytes()),
223-
self.layout.align.min(scratch_align));
223+
self.layout.align.min(scratch_align),
224+
MemFlags::empty());
224225

225226
bx.lifetime_end(llscratch, scratch_size);
226227
}

‎src/librustc_trans/base.rs‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ use rustc_incremental;
5353
use allocator;
5454
use mir::place::PlaceRef;
5555
use attributes;
56-
use builder::Builder;
56+
use builder::{Builder, MemFlags};
5757
use callee;
5858
use common::{C_bool, C_bytes_in_context, C_i32, C_usize};
5959
use rustc_mir::monomorphize::collector::{self, MonoItemCollectionMode};
@@ -320,7 +320,7 @@ pub fn coerce_unsized_into<'a, 'tcx>(bx: &Builder<'a, 'tcx>,
320320

321321
if src_f.layout.ty == dst_f.layout.ty {
322322
memcpy_ty(bx, dst_f.llval, src_f.llval, src_f.layout,
323-
src_f.align.min(dst_f.align));
323+
src_f.align.min(dst_f.align), MemFlags::empty());
324324
} else {
325325
coerce_unsized_into(bx, src_f, dst_f);
326326
}
@@ -408,7 +408,15 @@ pub fn call_memcpy(bx: &Builder,
408408
dst: ValueRef,
409409
src: ValueRef,
410410
n_bytes: ValueRef,
411-
align: Align) {
411+
align: Align,
412+
flags: MemFlags) {
413+
if flags.contains(MemFlags::NONTEMPORAL) {
414+
// HACK(nox): This is inefficient but there is no nontemporal memcpy.
415+
let val = bx.load(src, align);
416+
let ptr = bx.pointercast(dst, val_ty(val).ptr_to());
417+
bx.store_with_flags(val, ptr, align, flags);
418+
return;
419+
}
412420
let cx = bx.cx;
413421
let ptr_width = &cx.sess().target.target.target_pointer_width;
414422
let key = format!("llvm.memcpy.p0i8.p0i8.i{}", ptr_width);
@@ -417,7 +425,7 @@ pub fn call_memcpy(bx: &Builder,
417425
let dst_ptr = bx.pointercast(dst, Type::i8p(cx));
418426
let size = bx.intcast(n_bytes, cx.isize_ty, false);
419427
let align = C_i32(cx, align.abi() as i32);
420-
let volatile = C_bool(cx, false);
428+
let volatile = C_bool(cx, flags.contains(MemFlags::VOLATILE));
421429
bx.call(memcpy, &[dst_ptr, src_ptr, size, align, volatile], None);
422430
}
423431

@@ -427,13 +435,14 @@ pub fn memcpy_ty<'a, 'tcx>(
427435
src: ValueRef,
428436
layout: TyLayout<'tcx>,
429437
align: Align,
438+
flags: MemFlags,
430439
) {
431440
let size = layout.size.bytes();
432441
if size == 0 {
433442
return;
434443
}
435444

436-
call_memcpy(bx, dst, src, C_usize(bx.cx, size), align);
445+
call_memcpy(bx, dst, src, C_usize(bx.cx, size), align, flags);
437446
}
438447

439448
pub fn call_memset<'a, 'tcx>(bx: &Builder<'a, 'tcx>,

‎src/librustc_trans/builder.rs‎

Lines changed: 30 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,13 @@ fn noname() -> *const c_char {
5050
&CNULL
5151
}
5252

53+
bitflags! {
54+
pub struct MemFlags: u8 {
55+
const VOLATILE = 1 << 0;
56+
const NONTEMPORAL = 1 << 1;
57+
}
58+
}
59+
5360
impl<'a, 'tcx> Builder<'a, 'tcx> {
5461
pub fn new_block<'b>(cx: &'a CodegenCx<'a, 'tcx>, llfn: ValueRef, name: &'b str) -> Self {
5562
let bx = Builder::with_cx(cx);
@@ -579,29 +586,39 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
579586
}
580587

581588
pub fn store(&self, val: ValueRef, ptr: ValueRef, align: Align) -> ValueRef {
582-
debug!("Store {:?} -> {:?}", Value(val), Value(ptr));
589+
self.store_with_flags(val, ptr, align, MemFlags::empty())
590+
}
591+
592+
pub fn store_with_flags(
593+
&self,
594+
val: ValueRef,
595+
ptr: ValueRef,
596+
align: Align,
597+
flags: MemFlags,
598+
) -> ValueRef {
599+
debug!("Store {:?} -> {:?} ({:?})", Value(val), Value(ptr), flags);
583600
assert!(!self.llbuilder.is_null());
584601
self.count_insn("store");
585602
let ptr = self.check_store(val, ptr);
586603
unsafe {
587604
let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
588605
llvm::LLVMSetAlignment(store, align.abi() as c_uint);
606+
if flags.contains(MemFlags::VOLATILE) {
607+
llvm::LLVMSetVolatile(store, llvm::True);
608+
}
609+
if flags.contains(MemFlags::NONTEMPORAL) {
610+
// According to LLVM [1] building a nontemporal store must
611+
// *always* point to a metadata value of the integer 1.
612+
//
613+
// [1]: http://llvm.org/docs/LangRef.html#store-instruction
614+
let one = C_i32(self.cx, 1);
615+
let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1);
616+
llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node);
617+
}
589618
store
590619
}
591620
}
592621

593-
pub fn volatile_store(&self, val: ValueRef, ptr: ValueRef) -> ValueRef {
594-
debug!("Store {:?} -> {:?}", Value(val), Value(ptr));
595-
assert!(!self.llbuilder.is_null());
596-
self.count_insn("store.volatile");
597-
let ptr = self.check_store(val, ptr);
598-
unsafe {
599-
let insn = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
600-
llvm::LLVMSetVolatile(insn, llvm::True);
601-
insn
602-
}
603-
}
604-
605622
pub fn atomic_store(&self, val: ValueRef, ptr: ValueRef,
606623
order: AtomicOrdering, align: Align) {
607624
debug!("Store {:?} -> {:?}", Value(val), Value(ptr));
@@ -615,29 +632,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
615632
}
616633
}
617634

618-
pub fn nontemporal_store(&self, val: ValueRef, ptr: ValueRef) -> ValueRef {
619-
debug!("Store {:?} -> {:?}", Value(val), Value(ptr));
620-
assert!(!self.llbuilder.is_null());
621-
self.count_insn("store.nontemporal");
622-
let ptr = self.check_store(val, ptr);
623-
unsafe {
624-
let insn = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
625-
626-
// According to LLVM [1] building a nontemporal store must *always*
627-
// point to a metadata value of the integer 1. Who knew?
628-
//
629-
// [1]: http://llvm.org/docs/LangRef.html#store-instruction
630-
let one = C_i32(self.cx, 1);
631-
let node = llvm::LLVMMDNodeInContext(self.cx.llcx,
632-
&one,
633-
1);
634-
llvm::LLVMSetMetadata(insn,
635-
llvm::MD_nontemporal as c_uint,
636-
node);
637-
insn
638-
}
639-
}
640-
641635
pub fn gep(&self, ptr: ValueRef, indices: &[ValueRef]) -> ValueRef {
642636
self.count_insn("gep");
643637
unsafe {

‎src/librustc_trans/intrinsic.rs‎

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -247,26 +247,8 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bx: &Builder<'a, 'tcx>,
247247
to_immediate(bx, load, cx.layout_of(tp_ty))
248248
},
249249
"volatile_store" => {
250-
let tp_ty = substs.type_at(0);
251250
let dst = args[0].deref(bx.cx);
252-
if let OperandValue::Pair(a, b) = args[1].val {
253-
bx.volatile_store(a, dst.project_field(bx, 0).llval);
254-
bx.volatile_store(b, dst.project_field(bx, 1).llval);
255-
} else {
256-
let val = if let OperandValue::Ref(ptr, align) = args[1].val {
257-
bx.load(ptr, align)
258-
} else {
259-
if dst.layout.is_zst() {
260-
return;
261-
}
262-
from_immediate(bx, args[1].immediate())
263-
};
264-
let ptr = bx.pointercast(dst.llval, val_ty(val).ptr_to());
265-
let store = bx.volatile_store(val, ptr);
266-
unsafe {
267-
llvm::LLVMSetAlignment(store, cx.align_of(tp_ty).abi() as u32);
268-
}
269-
}
251+
args[1].val.volatile_store(bx, dst);
270252
return;
271253
},
272254
"prefetch_read_data" | "prefetch_write_data" |
@@ -551,19 +533,9 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bx: &Builder<'a, 'tcx>,
551533
}
552534

553535
"nontemporal_store" => {
554-
let tp_ty = substs.type_at(0);
555536
let dst = args[0].deref(bx.cx);
556-
let val = if let OperandValue::Ref(ptr, align) = args[1].val {
557-
bx.load(ptr, align)
558-
} else {
559-
from_immediate(bx, args[1].immediate())
560-
};
561-
let ptr = bx.pointercast(dst.llval, val_ty(val).ptr_to());
562-
let store = bx.nontemporal_store(val, ptr);
563-
unsafe {
564-
llvm::LLVMSetAlignment(store, cx.align_of(tp_ty).abi() as u32);
565-
}
566-
return
537+
args[1].val.nontemporal_store(bx, dst);
538+
return;
567539
}
568540

569541
_ => {

‎src/librustc_trans/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
use rustc::dep_graph::WorkProduct;
3535
use syntax_pos::symbol::Symbol;
3636

37+
#[macro_use] extern crate bitflags;
3738
extern crate flate2;
3839
extern crate libc;
3940
#[macro_use] extern crate rustc;

‎src/librustc_trans/mir/block.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use rustc::mir::interpret::EvalErrorKind;
1717
use abi::{Abi, ArgType, ArgTypeExt, FnType, FnTypeExt, LlvmType, PassMode};
1818
use base;
1919
use callee;
20-
use builder::Builder;
20+
use builder::{Builder, MemFlags};
2121
use common::{self, C_bool, C_str_slice, C_struct, C_u32, C_uint_big, C_undef};
2222
use consts;
2323
use meth;
@@ -626,7 +626,7 @@ impl<'a, 'tcx> FunctionCx<'a, 'tcx> {
626626
// have scary latent bugs around.
627627

628628
let scratch = PlaceRef::alloca(bx, arg.layout, "arg");
629-
base::memcpy_ty(bx, scratch.llval, llval, op.layout, align);
629+
base::memcpy_ty(bx, scratch.llval, llval, op.layout, align, MemFlags::empty());
630630
(scratch.llval, scratch.align, true)
631631
} else {
632632
(llval, align, true)

‎src/librustc_trans/mir/operand.rs‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ use rustc_data_structures::indexed_vec::Idx;
1818

1919
use base;
2020
use common::{self, CodegenCx, C_null, C_undef, C_usize};
21-
use builder::Builder;
21+
use builder::{Builder, MemFlags};
2222
use value::Value;
2323
use type_of::LayoutLlvmExt;
2424
use type_::Type;
@@ -272,18 +272,32 @@ impl<'a, 'tcx> OperandRef<'tcx> {
272272

273273
impl<'a, 'tcx> OperandValue {
274274
pub fn store(self, bx: &Builder<'a, 'tcx>, dest: PlaceRef<'tcx>) {
275+
self.store_with_flags(bx, dest, MemFlags::empty());
276+
}
277+
278+
pub fn volatile_store(self, bx: &Builder<'a, 'tcx>, dest: PlaceRef<'tcx>) {
279+
self.store_with_flags(bx, dest, MemFlags::VOLATILE);
280+
}
281+
282+
pub fn nontemporal_store(self, bx: &Builder<'a, 'tcx>, dest: PlaceRef<'tcx>) {
283+
self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL);
284+
}
285+
286+
fn store_with_flags(self, bx: &Builder<'a, 'tcx>, dest: PlaceRef<'tcx>, flags: MemFlags) {
275287
debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
276288
// Avoid generating stores of zero-sized values, because the only way to have a zero-sized
277289
// value is through `undef`, and store itself is useless.
278290
if dest.layout.is_zst() {
279291
return;
280292
}
281293
match self {
282-
OperandValue::Ref(r, source_align) =>
294+
OperandValue::Ref(r, source_align) => {
283295
base::memcpy_ty(bx, dest.llval, r, dest.layout,
284-
source_align.min(dest.align)),
296+
source_align.min(dest.align), flags)
297+
}
285298
OperandValue::Immediate(s) => {
286-
bx.store(base::from_immediate(bx, s), dest.llval, dest.align);
299+
let val = base::from_immediate(bx, s);
300+
bx.store_with_flags(val, dest.llval, dest.align, flags);
287301
}
288302
OperandValue::Pair(a, b) => {
289303
for (i, &x) in [a, b].iter().enumerate() {
@@ -292,7 +306,8 @@ impl<'a, 'tcx> OperandValue {
292306
if common::val_ty(x) == Type::i1(bx.cx) {
293307
llptr = bx.pointercast(llptr, Type::i8p(bx.cx));
294308
}
295-
bx.store(base::from_immediate(bx, x), llptr, dest.align);
309+
let val = base::from_immediate(bx, x);
310+
bx.store_with_flags(val, llptr, dest.align, flags);
296311
}
297312
}
298313
}

0 commit comments

Comments
 (0)