cmov: add cmovz/cmovnz-alike support for aarch64 (#744)

Author: codahale

cmov/Cargo.toml

@@ -3,7 +3,7 @@ name = "cmov"
 description = """
 Conditional move CPU intrinsics which are guaranteed to execute in
 constant-time and not be rewritten as branches by the compiler.
-Provides wrappers for the CMOV family of instructions on x86/x86_64 CPUs.
+Provides wrappers for the CMOV family of instructions on x86/x86_64/aarch64 CPUs.
 """
 version = "0.1.0"
 authors = ["RustCrypto Developers"]

cmov/README.md

@@ -9,7 +9,7 @@
 Conditional move CPU intrinsics which are guaranteed to execute in
 constant-time and not be rewritten as branches by the compiler.
 
-Provides wrappers for the [CMOV family] of instructions on x86/x86_64 CPUs.
+Provides wrappers for the [CMOV family] of instructions on x86/x86_64 and AArch64 CPUs.
 
 [Documentation][docs-link]
 
@@ -36,8 +36,9 @@ lowerings, such as the [x86-cmov-conversion] pass.
 This crate provides guaranteed constant-time operation using inline assembly
 on the following CPU architectures:
 
-- [x] `x86`
-- [x] `x86_64`
+- [x] `x86` (`CMOVZ`, `CMOVNZ`)
+- [x] `x86_64` (`CMOVZ`, CMOVNZ`)
+- [x] `aarch64` (`CSEL`)
 
 On other target architectures, a "best effort" portable fallback implementation
 based on bitwise arithmetic is used instead. However, we cannot guarantee that

cmov/src/lib.rs

@@ -6,7 +6,7 @@
 )]
 #![warn(missing_docs, rust_2018_idioms, unused_qualifications)]
 
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
 use core::arch::asm;
 
 /// Move if zero.
@@ -28,6 +28,46 @@ pub fn cmovz(condition: usize, src: usize, dst: &mut usize) {
     }
 }
 
+/// Move if zero.
+///
+/// Uses a `cmp` instruction to check if the given `condition` value is
+/// equal to zero, then calls `csel` to conditionally move
+/// `src` to `dst` when `condition` is equal to zero.
+#[cfg(any(target_arch = "aarch64"))]
+#[inline(always)]
+pub fn cmovz(condition: usize, src: usize, dst: &mut usize) {
+    unsafe {
+        asm! {
+            "cmp {0}, 0",
+            "csel {1}, {2}, {3}, EQ",
+            in(reg) condition,
+            inlateout(reg) *dst,
+            in(reg) src,
+            in(reg) *dst,
+        };
+    }
+}
+
+/// Move if not zero.
+///
+/// Uses a `cmp` instruction to check if the given `condition` value is not
+/// equal to zero, then calls `csel` to conditionally move
+/// `src` to `dst` when `condition` is nonzero.
+#[cfg(any(target_arch = "aarch64"))]
+#[inline(always)]
+pub fn cmovnz(condition: usize, src: usize, dst: &mut usize) {
+    unsafe {
+        asm! {
+            "cmp {0}, 0",
+            "csel {1}, {2}, {3}, NE",
+            in(reg) condition,
+            inlateout(reg) *dst,
+            in(reg) src,
+            in(reg) *dst,
+        };
+    }
+}
+
 /// Move if not zero.
 ///
 /// Uses a `test` instruction to check if the given `condition` value is not
@@ -52,7 +92,7 @@ pub fn cmovnz(condition: usize, src: usize, dst: &mut usize) {
 /// This implementation is based on portable bitwise arithmetic but cannot
 /// guarantee that the resulting generated assembly is free of branch
 /// instructions.
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
+#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
 #[inline(never)]
 pub fn cmovz(condition: usize, src: usize, dst: &mut usize) {
     let mask = (1 ^ is_non_zero(condition)).wrapping_sub(1);
@@ -64,7 +104,7 @@ pub fn cmovz(condition: usize, src: usize, dst: &mut usize) {
 /// This implementation is based on portable bitwise arithmetic but cannot
 /// guarantee that the resulting generated assembly is free of branch
 /// instructions.
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
+#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
 #[inline(never)]
 pub fn cmovnz(condition: usize, src: usize, dst: &mut usize) {
     let mask = is_non_zero(condition).wrapping_sub(1);
@@ -76,7 +116,7 @@ pub fn cmovnz(condition: usize, src: usize, dst: &mut usize) {
 /// # Returns
 /// - `condition` is zero: `0`
 /// - `condition` is non-zero: `1`
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
+#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
 #[inline(always)]
 fn is_non_zero(condition: usize) -> usize {
     const SHIFT_BITS: usize = core::mem::size_of::<usize>() - 1;