Reimplement aarch64 vld1* instructions to not cause individual loads under certain circumstances.

Author: hkratz

crates/core_arch/src/aarch64/neon/mod.rs

@@ -464,326 +464,191 @@ pub unsafe fn vcopy_laneq_f64<const LANE1: i32, const LANE2: i32>(
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_s8(ptr: *const i8) -> int8x8_t {
-    transmute(i8x8::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-    ))
+    core::ptr::read_unaligned(ptr as *const int8x8_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_s8(ptr: *const i8) -> int8x16_t {
-    transmute(i8x16::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-        *ptr.offset(8),
-        *ptr.offset(9),
-        *ptr.offset(10),
-        *ptr.offset(11),
-        *ptr.offset(12),
-        *ptr.offset(13),
-        *ptr.offset(14),
-        *ptr.offset(15),
-    ))
+    core::ptr::read_unaligned(ptr as *const int8x16_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_s16(ptr: *const i16) -> int16x4_t {
-    transmute(i16x4::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-    ))
+    core::ptr::read_unaligned(ptr as *const int16x4_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_s16(ptr: *const i16) -> int16x8_t {
-    transmute(i16x8::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-    ))
+    core::ptr::read_unaligned(ptr as *const int16x8_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_s32(ptr: *const i32) -> int32x2_t {
-    transmute(i32x2::new(*ptr, *ptr.offset(1)))
+    core::ptr::read_unaligned(ptr as *const int32x2_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_s32(ptr: *const i32) -> int32x4_t {
-    transmute(i32x4::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-    ))
+    core::ptr::read_unaligned(ptr as *const int32x4_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_s64(ptr: *const i64) -> int64x1_t {
-    transmute(i64x1::new(*ptr))
+    core::ptr::read_unaligned(ptr as *const int64x1_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_s64(ptr: *const i64) -> int64x2_t {
-    transmute(i64x2::new(*ptr, *ptr.offset(1)))
+    core::ptr::read_unaligned(ptr as *const int64x2_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_u8(ptr: *const u8) -> uint8x8_t {
-    transmute(u8x8::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-    ))
+    core::ptr::read_unaligned(ptr as *const uint8x8_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_u8(ptr: *const u8) -> uint8x16_t {
-    transmute(u8x16::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-        *ptr.offset(8),
-        *ptr.offset(9),
-        *ptr.offset(10),
-        *ptr.offset(11),
-        *ptr.offset(12),
-        *ptr.offset(13),
-        *ptr.offset(14),
-        *ptr.offset(15),
-    ))
+    core::ptr::read_unaligned(ptr as *const uint8x16_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_u16(ptr: *const u16) -> uint16x4_t {
-    transmute(u16x4::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-    ))
+    core::ptr::read_unaligned(ptr as *const uint16x4_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_u16(ptr: *const u16) -> uint16x8_t {
-    transmute(u16x8::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-    ))
+    core::ptr::read_unaligned(ptr as *const uint16x8_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_u32(ptr: *const u32) -> uint32x2_t {
-    transmute(u32x2::new(*ptr, *ptr.offset(1)))
+    core::ptr::read_unaligned(ptr as *const uint32x2_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_u32(ptr: *const u32) -> uint32x4_t {
-    transmute(u32x4::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-    ))
+    core::ptr::read_unaligned(ptr as *const uint32x4_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_u64(ptr: *const u64) -> uint64x1_t {
-    transmute(u64x1::new(*ptr))
+    core::ptr::read_unaligned(ptr as *const uint64x1_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_u64(ptr: *const u64) -> uint64x2_t {
-    transmute(u64x2::new(*ptr, *ptr.offset(1)))
+    core::ptr::read_unaligned(ptr as *const uint64x2_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_p8(ptr: *const p8) -> poly8x8_t {
-    transmute(u8x8::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-    ))
+    core::ptr::read_unaligned(ptr as *const poly8x8_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_p8(ptr: *const p8) -> poly8x16_t {
-    transmute(u8x16::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-        *ptr.offset(8),
-        *ptr.offset(9),
-        *ptr.offset(10),
-        *ptr.offset(11),
-        *ptr.offset(12),
-        *ptr.offset(13),
-        *ptr.offset(14),
-        *ptr.offset(15),
-    ))
+    core::ptr::read_unaligned(ptr as *const poly8x16_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_p16(ptr: *const p16) -> poly16x4_t {
-    transmute(u16x4::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-    ))
+    core::ptr::read_unaligned(ptr as *const poly16x4_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_p16(ptr: *const p16) -> poly16x8_t {
-    transmute(u16x8::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-        *ptr.offset(4),
-        *ptr.offset(5),
-        *ptr.offset(6),
-        *ptr.offset(7),
-    ))
+    core::ptr::read_unaligned(ptr as *const poly16x8_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_f32(ptr: *const f32) -> float32x2_t {
-    transmute(f32x2::new(*ptr, *ptr.offset(1)))
+    core::ptr::read_unaligned(ptr as *const float32x2_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_f32(ptr: *const f32) -> float32x4_t {
-    transmute(f32x4::new(
-        *ptr,
-        *ptr.offset(1),
-        *ptr.offset(2),
-        *ptr.offset(3),
-    ))
+    core::ptr::read_unaligned(ptr as *const float32x4_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1_f64(ptr: *const f64) -> float64x1_t {
-    transmute(f64x1::new(*ptr))
+    core::ptr::read_unaligned(ptr as *const float64x1_t)
 }
 
 /// Load multiple single-element structures to one, two, three, or four registers.
 #[inline]
 #[target_feature(enable = "neon")]
 #[cfg_attr(test, assert_instr(ldr))]
 pub unsafe fn vld1q_f64(ptr: *const f64) -> float64x2_t {
-    transmute(f64x2::new(*ptr, *ptr.offset(1)))
+    core::ptr::read_unaligned(ptr as *const float64x2_t)
 }
 
 /// Store multiple single-element structures from one, two, three, or four registers.