Skip to content

Port the remaining SIMD intrinsics to const-eval #147520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 3, 2025
+450 −218
Merged

Port the remaining SIMD intrinsics to const-eval #147520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c4d6b0b
Implement `simd_fma` and `simd_relaxed_fma` in const-eval
sayantn Oct 9, 2025
d335ea9
Refactor implementation of float minmax intrinsics
sayantn Oct 9, 2025
ebf2e10
Refactor float rounding intrinsics a bit
sayantn Oct 10, 2025
4d70bf4
Add some missing f16-f128 support in intrinsics
sayantn Oct 10, 2025
0681bae
Add Miri tests for f16/f128 SIMD operations
sayantn Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 124 additions & 97 deletions compiler/rustc_const_eval/src/interpret/intrinsics.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,35 @@ use super::{
};
use crate::fluent_generated as fluent;

#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum MulAddType {
/// Used with `fma` and `simd_fma`, always uses fused-multiply-add
Fused,
/// Used with `fmuladd` and `simd_relaxed_fma`, nondeterministically determines whether to use
/// fma or simple multiply-add
Nondeterministic,
}

#[derive(Copy, Clone)]
pub(crate) enum MinMax {
/// The IEEE `Minimum` operation - see `f32::minimum` etc
/// In particular, `-0.0` is considered smaller than `+0.0` and
/// if either input is NaN, the result is NaN.
Minimum,
/// The IEEE `MinNum` operation - see `f32::min` etc
/// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic,
/// and is one argument is NaN, the other one is returned.
MinNum,
/// The IEEE `Maximum` operation - see `f32::maximum` etc
/// In particular, `-0.0` is considered smaller than `+0.0` and
/// if either input is NaN, the result is NaN.
Maximum,
/// The IEEE `MaxNum` operation - see `f32::max` etc
/// In particular, if the inputs are `-0.0` and `+0.0`, the result is non-deterministic,
/// and is one argument is NaN, the other one is returned.
MaxNum,
}

/// Directly returns an `Allocation` containing an absolute path representation of the given type.
pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
let path = crate::util::type_name(tcx, ty);
Expand Down Expand Up @@ -504,25 +533,33 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
}

sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
sym::minnumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::MinNum, dest)?,
sym::minnumf32 => self.float_minmax_intrinsic::<Single>(args, MinMax::MinNum, dest)?,
sym::minnumf64 => self.float_minmax_intrinsic::<Double>(args, MinMax::MinNum, dest)?,
sym::minnumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::MinNum, dest)?,

sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
sym::minimumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Minimum, dest)?,
sym::minimumf32 => {
self.float_minmax_intrinsic::<Single>(args, MinMax::Minimum, dest)?
}
sym::minimumf64 => {
self.float_minmax_intrinsic::<Double>(args, MinMax::Minimum, dest)?
}
sym::minimumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Minimum, dest)?,

sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
sym::maxnumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::MaxNum, dest)?,
sym::maxnumf32 => self.float_minmax_intrinsic::<Single>(args, MinMax::MaxNum, dest)?,
sym::maxnumf64 => self.float_minmax_intrinsic::<Double>(args, MinMax::MaxNum, dest)?,
sym::maxnumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::MaxNum, dest)?,

sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
sym::maximumf16 => self.float_minmax_intrinsic::<Half>(args, MinMax::Maximum, dest)?,
sym::maximumf32 => {
self.float_minmax_intrinsic::<Single>(args, MinMax::Maximum, dest)?
}
sym::maximumf64 => {
self.float_minmax_intrinsic::<Double>(args, MinMax::Maximum, dest)?
}
sym::maximumf128 => self.float_minmax_intrinsic::<Quad>(args, MinMax::Maximum, dest)?,

sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
Expand Down Expand Up @@ -630,14 +667,22 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
dest,
rustc_apfloat::Round::NearestTiesToEven,
)?,
sym::fmaf16 => self.fma_intrinsic::<Half>(args, dest)?,
sym::fmaf32 => self.fma_intrinsic::<Single>(args, dest)?,
sym::fmaf64 => self.fma_intrinsic::<Double>(args, dest)?,
sym::fmaf128 => self.fma_intrinsic::<Quad>(args, dest)?,
sym::fmuladdf16 => self.float_muladd_intrinsic::<Half>(args, dest)?,
sym::fmuladdf32 => self.float_muladd_intrinsic::<Single>(args, dest)?,
sym::fmuladdf64 => self.float_muladd_intrinsic::<Double>(args, dest)?,
sym::fmuladdf128 => self.float_muladd_intrinsic::<Quad>(args, dest)?,
sym::fmaf16 => self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Fused)?,
sym::fmaf32 => self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Fused)?,
sym::fmaf64 => self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Fused)?,
sym::fmaf128 => self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Fused)?,
sym::fmuladdf16 => {
self.float_muladd_intrinsic::<Half>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf32 => {
self.float_muladd_intrinsic::<Single>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf64 => {
self.float_muladd_intrinsic::<Double>(args, dest, MulAddType::Nondeterministic)?
}
sym::fmuladdf128 => {
self.float_muladd_intrinsic::<Quad>(args, dest, MulAddType::Nondeterministic)?
}

// Unsupported intrinsic: skip the return_to_block below.
_ => return interp_ok(false),
Expand Down Expand Up @@ -919,76 +964,45 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
}

fn float_min_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = if a == b {
// They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
// Let the machine decide which one to return.
M::equal_float_min_max(self, a, b)
} else {
self.adjust_nan(a.min(b), &[a, b])
};
self.write_scalar(res, dest)?;
interp_ok(())
}

fn float_max_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
fn float_minmax<F>(
&self,
a: Scalar<M::Provenance>,
b: Scalar<M::Provenance>,
op: MinMax,
) -> InterpResult<'tcx, Scalar<M::Provenance>>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = if a == b {
let a: F = a.to_float()?;
let b: F = b.to_float()?;
let res = if matches!(op, MinMax::MinNum | MinMax::MaxNum) && a == b {
// They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
// Let the machine decide which one to return.
M::equal_float_min_max(self, a, b)
} else {
self.adjust_nan(a.max(b), &[a, b])
let result = match op {
MinMax::Minimum => a.minimum(b),
MinMax::MinNum => a.min(b),
MinMax::Maximum => a.maximum(b),
MinMax::MaxNum => a.max(b),
};
self.adjust_nan(result, &[a, b])
};
self.write_scalar(res, dest)?;
interp_ok(())
}

fn float_minimum_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = a.minimum(b);
let res = self.adjust_nan(res, &[a, b]);
self.write_scalar(res, dest)?;
interp_ok(())
interp_ok(res.into())
}

fn float_maximum_intrinsic<F>(
fn float_minmax_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
op: MinMax,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let res = a.maximum(b);
let res = self.adjust_nan(res, &[a, b]);
let res =
self.float_minmax::<F>(self.read_scalar(&args[0])?, self.read_scalar(&args[1])?, op)?;
self.write_scalar(res, dest)?;
interp_ok(())
}
Expand Down Expand Up @@ -1022,56 +1036,69 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
interp_ok(())
}

fn float_round_intrinsic<F>(
fn float_round<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
x: Scalar<M::Provenance>,
mode: rustc_apfloat::Round,
) -> InterpResult<'tcx, ()>
) -> InterpResult<'tcx, Scalar<M::Provenance>>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let x: F = self.read_scalar(&args[0])?.to_float()?;
let x: F = x.to_float()?;
let res = x.round_to_integral(mode).value;
let res = self.adjust_nan(res, &[x]);
self.write_scalar(res, dest)?;
interp_ok(())
interp_ok(res.into())
}

fn fma_intrinsic<F>(
fn float_round_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
mode: rustc_apfloat::Round,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let c: F = self.read_scalar(&args[2])?.to_float()?;

let res = a.mul_add(b, c).value;
let res = self.adjust_nan(res, &[a, b, c]);
let res = self.float_round::<F>(self.read_scalar(&args[0])?, mode)?;
self.write_scalar(res, dest)?;
interp_ok(())
}

fn float_muladd<F>(
&self,
a: Scalar<M::Provenance>,
b: Scalar<M::Provenance>,
c: Scalar<M::Provenance>,
typ: MulAddType,
) -> InterpResult<'tcx, Scalar<M::Provenance>>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = a.to_float()?;
let b: F = b.to_float()?;
let c: F = c.to_float()?;

let fuse = typ == MulAddType::Fused || M::float_fuse_mul_add(self);

let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
let res = self.adjust_nan(res, &[a, b, c]);
interp_ok(res.into())
}

fn float_muladd_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &PlaceTy<'tcx, M::Provenance>,
typ: MulAddType,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
{
let a: F = self.read_scalar(&args[0])?.to_float()?;
let b: F = self.read_scalar(&args[1])?.to_float()?;
let c: F = self.read_scalar(&args[2])?.to_float()?;

let fuse = M::float_fuse_mul_add(self);
let a = self.read_scalar(&args[0])?;
let b = self.read_scalar(&args[1])?;
let c = self.read_scalar(&args[2])?;

let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value };
let res = self.adjust_nan(res, &[a, b, c]);
let res = self.float_muladd::<F>(a, b, c, typ)?;
self.write_scalar(res, dest)?;
interp_ok(())
}
Expand Down
Loading
Loading