From 4f24d142d960350f9cb585795c5cd58857782045 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 25 May 2025 18:51:16 +0000 Subject: [PATCH 01/38] Merge commit '979dcf8e2f213e4f4b645cb62e7fe9f4f2c0c785' into sync_cg_clif-2025-05-25 --- .github/workflows/main.yml | 6 +- Cargo.lock | 80 ++--- Cargo.toml | 24 +- Readme.md | 4 +- build_system/build_backend.rs | 2 +- build_system/build_sysroot.rs | 5 +- docs/usage.md | 2 +- example/mini_core.rs | 20 +- example/std_example.rs | 3 + ...0027-stdlib-128bit-atomic-operations.patch | 9 + rust-toolchain | 2 +- scripts/setup_rust_fork.sh | 2 +- scripts/test_rustc_tests.sh | 17 +- src/abi/mod.rs | 66 ++-- src/abi/pass_mode.rs | 2 + src/abi/returning.rs | 12 +- src/base.rs | 69 +++-- src/cast.rs | 21 +- src/codegen_f16_f128.rs | 284 ++++++++++++++++++ src/common.rs | 8 +- src/compiler_builtins.rs | 88 +++++- src/debuginfo/emit.rs | 39 ++- src/debuginfo/unwind.rs | 20 +- src/driver/jit.rs | 7 +- src/intrinsics/llvm.rs | 7 +- src/intrinsics/llvm_aarch64.rs | 9 +- src/intrinsics/llvm_x86.rs | 13 +- src/intrinsics/mod.rs | 181 ++++++++++- src/intrinsics/simd.rs | 36 ++- src/lib.rs | 43 ++- src/num.rs | 76 ++++- src/pretty_clif.rs | 73 +++-- src/trap.rs | 38 --- src/unwind_module.rs | 24 +- src/value_and_place.rs | 76 +++-- triagebot.toml | 7 + 36 files changed, 1110 insertions(+), 265 deletions(-) create mode 100644 src/codegen_f16_f128.rs delete mode 100644 src/trap.rs create mode 100644 triagebot.toml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6fd288d195c05..d92e0fdce99a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,9 +34,9 @@ jobs: - name: Rustfmt run: | cargo fmt --check - rustfmt --check build_system/main.rs - rustfmt --check example/* - rustfmt --check scripts/*.rs + rustfmt --check --edition 2024 build_system/main.rs + rustfmt --check --edition 2024 example/* + rustfmt --check --edition 2024 scripts/*.rs test: diff --git a/Cargo.lock b/Cargo.lock index e5f1896b9230b..a906bec8b7e23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,39 +43,42 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-assembler-x64" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4b56ebe316895d3fa37775d0a87b0c889cc933f5c8b253dbcc7c7bcb7fe7e4" +checksum = "9ff8e35182c7372df00447cb90a04e584e032c42b9b9b6e8c50ddaaf0d7900d5" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cabbc01dfbd7dcd6c329ca44f0212910309c221797ac736a67a5bc8857fe1b" +checksum = "14220f9c2698015c3b94dc6b84ae045c1c45509ddc406e43c6139252757fdb7a" +dependencies = [ + "cranelift-srcgen", +] [[package]] name = "cranelift-bforest" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ffe46df300a45f1dc6f609dc808ce963f0e3a2e971682c479a2d13e3b9b8ef" +checksum = "d372ef2777ceefd75829e1390211ac240e9196bc60699218f7ea2419038288ee" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b265bed7c51e1921fdae6419791d31af77d33662ee56d7b0fa0704dc8d231cab" +checksum = "56323783e423818fa89ce8078e90a3913d2a6e0810399bfce8ebd7ee87baa81f" [[package]] name = "cranelift-codegen" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e606230a7e3a6897d603761baee0d19f88d077f17b996bb5089488a29ae96e41" +checksum = "74ffb780aab6186c6e9ba26519654b1ac55a09c0a866f6088a4efbbd84da68ed" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -98,43 +101,44 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a63bffafc23bc60969ad528e138788495999d935f0adcfd6543cb151ca8637d" +checksum = "c23ef13814d3b39c869650d5961128cbbecad83fbdff4e6836a03ecf6862d7ed" dependencies = [ - "cranelift-assembler-x64", + "cranelift-assembler-x64-meta", "cranelift-codegen-shared", + "cranelift-srcgen", ] [[package]] name = "cranelift-codegen-shared" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af50281b67324b58e843170a6a5943cf6d387c06f7eeacc9f5696e4ab7ae7d7e" +checksum = "b9f623300657679f847803ce80811454bfff89cea4f6bf684be5c468d4a73631" [[package]] name = "cranelift-control" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c20c1b38d1abfbcebb0032e497e71156c0e3b8dcb3f0a92b9863b7bcaec290c" +checksum = "31f4168af69989aa6b91fab46799ed4df6096f3209f4a6c8fb4358f49c60188f" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2c67d95507c51b4a1ff3f3555fe4bfec36b9e13c1b684ccc602736f5d5f4a2" +checksum = "ca6fa9bae1c8de26d71ac2162f069447610fd91e7780cb480ee0d76ac81eabb8" dependencies = [ "cranelift-bitset", ] [[package]] name = "cranelift-frontend" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e002691cc69c38b54fc7ec93e5be5b744f627d027031d991cc845d1d512d0ce" +checksum = "b8219205608aa0b0e6769b580284a7e055c7e0c323c1041cde7ca078add3e412" dependencies = [ "cranelift-codegen", "log", @@ -144,15 +148,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93588ed1796cbcb0e2ad160403509e2c5d330d80dd6e0014ac6774c7ebac496" +checksum = "588d0c5964f10860b04043e55aab26d7f7a206b0fd4f10c5260e8aa5773832bd" [[package]] name = "cranelift-jit" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6682f0b193d6b7873cc8e7ed67e8776a8a26f50eeabf88534e9be618b9a03" +checksum = "56bd917ddc524f84f4066f954062875bdfc0dffea068ee94e906d98de5ac7c33" dependencies = [ "anyhow", "cranelift-codegen", @@ -170,9 +174,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff19784c6de05116e63e6a34791012bd927b2a4eac56233039c46f1b6a4edac8" +checksum = "68a03c057d8a992e06596c871341e446af43ff9224f941e5b8adea39137a5391" dependencies = [ "anyhow", "cranelift-codegen", @@ -181,9 +185,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5b09bdd6407bf5d89661b80cf926ce731c9e8cc184bf49102267a2369a8358e" +checksum = "19ed3c94cb97b14f92b6a94a1d45ef8c851f6a2ad9114e5d91d233f7da638fed" dependencies = [ "cranelift-codegen", "libc", @@ -192,9 +196,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.118.0" +version = "0.120.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685e8661a30d1cb69509f589ac643adeee79c5f63c0da316431b9fad29e6d3b4" +checksum = "a64dacef362a69375a604f6636e5e9a174fb96dba3b273646fcd9fa85c1d0997" dependencies = [ "anyhow", "cranelift-codegen", @@ -205,6 +209,12 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "cranelift-srcgen" +version = "0.120.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85256fac1519a7d25a040c1d850fba67478f3f021ad5fdf738ba4425ee862dbf" + [[package]] name = "crc32fast" version = "1.4.2" @@ -331,9 +341,9 @@ dependencies = [ [[package]] name = "regalloc2" -version = "0.11.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145c1c267e14f20fb0f88aa76a1c5ffec42d592c1d28b3cd9148ae35916158d3" +checksum = "5216b1837de2149f8bc8e6d5f88a9326b63b8c836ed58ce4a0a29ec736a59734" dependencies = [ "allocator-api2", "bumpalo", @@ -436,9 +446,9 @@ checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "wasmtime-jit-icache-coherence" -version = "31.0.0" +version = "33.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a54f6c6c7e9d7eeee32dfcc10db7f29d505ee7dd28d00593ea241d5f70698e64" +checksum = "175e924dbc944c185808466d1e90b5a7feb610f3b9abdfe26f8ee25fd1086d1c" dependencies = [ "anyhow", "cfg-if", diff --git a/Cargo.toml b/Cargo.toml index 08b60de14c1f8..94fcbd0a5023d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,12 +8,12 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.118.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } -cranelift-frontend = { version = "0.118.0" } -cranelift-module = { version = "0.118.0" } -cranelift-native = { version = "0.118.0" } -cranelift-jit = { version = "0.118.0", optional = true } -cranelift-object = { version = "0.118.0" } +cranelift-codegen = { version = "0.120.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } +cranelift-frontend = { version = "0.120.0" } +cranelift-module = { version = "0.120.0" } +cranelift-native = { version = "0.120.0" } +cranelift-jit = { version = "0.120.0", optional = true } +cranelift-object = { version = "0.120.0" } target-lexicon = "0.13" gimli = { version = "0.31", default-features = false, features = ["write"] } object = { version = "0.36", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } @@ -24,12 +24,12 @@ smallvec = "1.8.1" [patch.crates-io] # Uncomment to use an unreleased version of cranelift -#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" } -#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" } -#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" } -#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" } -#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" } -#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" } +#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } +#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } +#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } +#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } +#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } +#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } # Uncomment to use local checkout of cranelift #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } diff --git a/Readme.md b/Readme.md index 28edb5795ce3c..4d1e4d843ffeb 100644 --- a/Readme.md +++ b/Readme.md @@ -7,7 +7,7 @@ If not please open an issue. ## Download using Rustup -The Cranelift codegen backend is distributed in nightly builds on Linux and x86_64 macOS. If you want to +The Cranelift codegen backend is distributed in nightly builds on Linux, macOS and x86_64 Windows. If you want to install it using Rustup, you can do that by running: ```bash @@ -79,7 +79,7 @@ For more docs on how to build and test see [build_system/usage.txt](build_system Not all targets are available as rustup component for nightly. See notes in the platform support matrix. [^xcoff]: XCOFF object file format is not supported. -[^no-rustup]: Not available as rustup component for nightly. You can build it yourself. +[^no-rustup]: Not available as [rustup component for nightly](https://rust-lang.github.io/rustup-components-history/). You can build it yourself. ## Usage diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs index 72bc422523d55..bf7cf1c0a346f 100644 --- a/build_system/build_backend.rs +++ b/build_system/build_backend.rs @@ -18,7 +18,7 @@ pub(crate) fn build_backend( let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs); let mut rustflags = rustflags_from_env("RUSTFLAGS"); - rustflags.push("-Zallow-features=rustc_private".to_owned()); + rustflags.push("-Zallow-features=rustc_private,f16,f128".to_owned()); rustflags_to_cmd_env(&mut cmd, "RUSTFLAGS", &rustflags); if env::var("CG_CLIF_EXPENSIVE_CHECKS").is_ok() { diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index a73e3c87d43d0..00955998e703d 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -168,7 +168,8 @@ fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget { let file_name_str = file.file_name().unwrap().to_str().unwrap(); if (file_name_str.contains("rustc_") && !file_name_str.contains("rustc_std_workspace_") - && !file_name_str.contains("rustc_demangle")) + && !file_name_str.contains("rustc_demangle") + && !file_name_str.contains("rustc_literal_escaper")) || file_name_str.contains("chalk") || file_name_str.contains("tracing") || file_name_str.contains("regex") @@ -234,7 +235,7 @@ fn build_clif_sysroot_for_triple( compiler.rustflags.extend(rustflags); let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); build_cmd.arg("--release"); - build_cmd.arg("--features").arg("backtrace panic-unwind compiler-builtins-no-f16-f128"); + build_cmd.arg("--features").arg("backtrace panic-unwind"); build_cmd.arg(format!("-Zroot-dir={}", STDLIB_SRC.to_path(dirs).display())); build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); diff --git a/docs/usage.md b/docs/usage.md index dbe36109f83e9..9dcfee4f535a7 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -47,7 +47,7 @@ These are a few functions that allow you to easily run rust code from the shell ```bash function jit_naked() { - echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=jit-mode-Cprefer-dynamic + echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=jit-mode -Cprefer-dynamic } function jit() { diff --git a/example/mini_core.rs b/example/mini_core.rs index 6e345b2a6fdf0..1dc799c0aeea8 100644 --- a/example/mini_core.rs +++ b/example/mini_core.rs @@ -521,10 +521,28 @@ fn panic_cannot_unwind() -> ! { } #[lang = "eh_personality"] -fn eh_personality() -> ! { +// FIXME personality signature depends on target +fn eh_personality( + _version: i32, + _actions: i32, + _exception_class: u64, + _exception_object: *mut (), + _context: *mut (), +) -> i32 { loop {} } +#[lang = "panic_in_cleanup"] +fn panic_in_cleanup() -> ! { + loop {} +} + +#[cfg(all(unix, not(target_vendor = "apple")))] +#[link(name = "gcc_s")] +extern "C" { + fn _Unwind_Resume(exc: *mut ()) -> !; +} + #[lang = "drop_in_place"] #[allow(unconditional_recursion)] pub unsafe fn drop_in_place(to_drop: *mut T) { diff --git a/example/std_example.rs b/example/std_example.rs index 2d9de2a5b8d6b..5d83066cffb88 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -166,6 +166,7 @@ fn main() { enum Never {} } + #[cfg(not(target_arch = "s390x"))] // s390x doesn't have vector instructions enabled by default foo(I64X2([0, 0])); transmute_wide_pointer(); @@ -203,9 +204,11 @@ fn rust_call_abi() { rust_call_abi_callee((1, 2)); } +#[cfg_attr(target_arch = "s390x", allow(dead_code))] #[repr(simd)] struct I64X2([i64; 2]); +#[cfg_attr(target_arch = "s390x", allow(dead_code))] #[allow(improper_ctypes_definitions)] extern "C" fn foo(_a: I64X2) {} diff --git a/patches/0027-stdlib-128bit-atomic-operations.patch b/patches/0027-stdlib-128bit-atomic-operations.patch index d7e3b11127c42..38bb43f8204b4 100644 --- a/patches/0027-stdlib-128bit-atomic-operations.patch +++ b/patches/0027-stdlib-128bit-atomic-operations.patch @@ -37,6 +37,15 @@ diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index bf2b6d59f88..d5ccce03bbf 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs +@@ -300,8 +300,6 @@ impl_atomic_primitive!(AtomicI32(i32), size("32"), align(4)); + impl_atomic_primitive!(AtomicU32(u32), size("32"), align(4)); + impl_atomic_primitive!(AtomicI64(i64), size("64"), align(8)); + impl_atomic_primitive!(AtomicU64(u64), size("64"), align(8)); +-impl_atomic_primitive!(AtomicI128(i128), size("128"), align(16)); +-impl_atomic_primitive!(AtomicU128(u128), size("128"), align(16)); + + #[cfg(target_pointer_width = "16")] + impl_atomic_primitive!(AtomicIsize(isize), size("ptr"), align(2)); @@ -3585,44 +3585,6 @@ pub const fn as_ptr(&self) -> *mut $int_type { 8, u64 AtomicU64 diff --git a/rust-toolchain b/rust-toolchain index ceff15b1180a7..af4bd6dc6b851 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-03-30" +channel = "nightly-2025-05-25" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index ca6426f2ba9d3..532702bb1a469 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -43,7 +43,7 @@ verbose-tests = false # disabled bootstrap will crash trying to copy llvm tools for the bootstrap # compiler. llvm-tools = false -std-features = ["panic-unwind", "compiler-builtins-no-f16-f128"] +std-features = ["panic-unwind"] EOF diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 95a4302b5e471..32c71f433b0ff 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -53,6 +53,8 @@ rm -r tests/run-make/split-debuginfo # same rm -r tests/run-make/target-specs # i686 not supported by Cranelift rm -r tests/run-make/mismatching-target-triples # same rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly +rm tests/ui/asm/global-asm-mono-sym-fn.rs # same +rm tests/ui/asm/naked-asm-mono-sym-fn.rs # same rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported rm tests/ui/simd/simd-bitmask-notpow2.rs # non-pow-of-2 simd vector sizes rm -r tests/run-make/embed-source-dwarf # embedding sources in debuginfo @@ -70,19 +72,13 @@ rm tests/ui/consts/precise-drop-with-coverage.rs rm tests/ui/issues/issue-85461.rs rm -r tests/ui/instrument-coverage/ -# missing f16/f128 support -rm tests/ui/half-open-range-patterns/half-open-range-pats-semantics.rs -rm tests/ui/asm/aarch64/type-f16.rs -rm tests/ui/float/conv-bits-runtime-const.rs -rm tests/ui/consts/const-eval/float_methods.rs -rm tests/ui/match/match-float.rs - # optimization tests # ================== rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations rm tests/ui/codegen/init-large-type.rs # same rm -r tests/run-make/fmt-write-bloat/ # tests an optimization rm tests/ui/statics/const_generics.rs # same +rm tests/ui/linking/executable-no-mangle-strip.rs # requires --gc-sections to work for statics # backend specific tests # ====================== @@ -96,6 +92,7 @@ rm -r tests/run-make/llvm-location-discriminator-limit-dummy-span # same rm tests/ui/abi/stack-protector.rs # requires stack protector support rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific +rm -r tests/ui/optimization-remark.rs # same rm -r tests/run-make/print-to-output # requires --print relocation-models # requires asm, llvm-ir and/or llvm-bc emit support @@ -118,7 +115,6 @@ rm tests/ui/mir/mir_raw_fat_ptr.rs # same rm tests/ui/consts/issue-33537.rs # same rm tests/ui/consts/const-mut-refs-crate.rs # same rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift -rm tests/ui/abi/simd-abi-checks-avx.rs # attempts to declare function with two different signatures # doesn't work due to the way the rustc test suite is invoked. # should work when using ./x.py test the way it is intended @@ -129,7 +125,11 @@ rm -r tests/run-make/compiler-builtins # Expects lib/rustlib/src/rust to contain rm -r tests/run-make/translation # same rm -r tests/run-make/missing-unstable-trait-bound # This disables support for unstable features, but running cg_clif needs some unstable features rm -r tests/run-make/const-trait-stable-toolchain # same +rm -r tests/run-make/print-request-help-stable-unstable # same rm -r tests/run-make/incr-add-rust-src-component +rm tests/ui/errors/remap-path-prefix-sysroot.rs # different sysroot source path +rm -r tests/run-make/export/extern-opt # something about rustc version mismatches +rm -r tests/run-make/export # same # genuine bugs # ============ @@ -141,6 +141,7 @@ rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue rm tests/ui/backtrace/synchronized-panic-handler.rs # missing needs-unwind annotation rm tests/ui/lint/non-snake-case/lint-non-snake-case-crate.rs # same +rm tests/ui/async-await/async-drop/async-drop-initial.rs # same (rust-lang/rust#140493) rm -r tests/ui/codegen/equal-pointers-unequal # make incorrect assumptions about the location of stack variables rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 5f631405a9a4a..5f7bf3821d77d 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -20,6 +20,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_session::Session; use rustc_span::source_map::Spanned; use rustc_target::callconv::{Conv, FnAbi, PassMode}; +use smallvec::SmallVec; use self::pass_mode::*; pub(crate) use self::returning::codegen_return; @@ -153,10 +154,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let ret = self.lib_call_unadjusted(name, params, returns, &args)[0]; - // FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128 - let ret_ptr = self.create_stack_slot(16, 16); - ret_ptr.store(self, ret, MemFlags::trusted()); - Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]) + Cow::Owned(vec![codegen_bitcast(self, types::I128, ret)]) } else if ret_single_i128 && self.tcx.sess.target.arch == "s390x" { // Return i128 using a return area pointer on s390x. let mut params = params; @@ -184,11 +182,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv }; let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); - if self.clif_comments.enabled() { - self.add_comment(func_ref, format!("{:?}", name)); - } let call_inst = self.bcx.ins().call(func_ref, args); if self.clif_comments.enabled() { + self.add_comment(func_ref, format!("{:?}", name)); self.add_comment(call_inst, format!("lib_call {}", name)); } let results = self.bcx.inst_results(call_inst); @@ -384,6 +380,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( args: &[Spanned>], destination: Place<'tcx>, target: Option, + _unwind: UnwindAction, ) { let func = codegen_operand(fx, func); let fn_sig = func.layout().ty.fn_sig(fx.tcx); @@ -529,7 +526,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( Some(Instance { def: InstanceKind::Virtual(_, idx), .. }) => { if fx.clif_comments.enabled() { let nop_inst = fx.bcx.ins().nop(); - fx.add_comment( + fx.add_post_comment( nop_inst, with_no_trimmed_paths!(format!( "virtual call; self arg pass mode: {:?}", @@ -555,7 +552,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( None => { if fx.clif_comments.enabled() { let nop_inst = fx.bcx.ins().nop(); - fx.add_comment(nop_inst, "indirect call"); + fx.add_post_comment(nop_inst, "indirect call"); } let func = func.load_scalar(fx); @@ -585,17 +582,18 @@ pub(crate) fn codegen_terminator_call<'tcx>( adjust_call_for_c_variadic(fx, &fn_abi, source_info, func_ref, &mut call_args); } - if fx.clif_comments.enabled() { - let nop_inst = fx.bcx.ins().nop(); - with_no_trimmed_paths!(fx.add_comment(nop_inst, format!("abi: {:?}", fn_abi))); - } - - match func_ref { + let call_inst = match func_ref { CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args), CallTarget::Indirect(sig, func_ptr) => { fx.bcx.ins().call_indirect(sig, func_ptr, &call_args) } + }; + + if fx.clif_comments.enabled() { + with_no_trimmed_paths!(fx.add_comment(call_inst, format!("abi: {:?}", fn_abi))); } + + fx.bcx.func.dfg.inst_results(call_inst).iter().copied().collect::>() }); if let Some(dest) = target { @@ -705,13 +703,16 @@ pub(crate) fn codegen_drop<'tcx>( source_info: mir::SourceInfo, drop_place: CPlace<'tcx>, target: BasicBlock, + _unwind: UnwindAction, ) { let ty = drop_place.layout().ty; let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty); + let ret_block = fx.get_block(target); // AsyncDropGlueCtorShim can't be here if let ty::InstanceKind::DropGlue(_, None) = drop_instance.def { // we don't actually need to drop anything + fx.bcx.ins().jump(ret_block, &[]); } else { match ty.kind() { ty::Dynamic(_, _, ty::Dyn) => { @@ -748,7 +749,9 @@ pub(crate) fn codegen_drop<'tcx>( let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); let sig = fx.bcx.import_signature(sig); + // FIXME implement cleanup on exceptions fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]); + fx.bcx.ins().jump(ret_block, &[]); } ty::Dynamic(_, _, ty::DynStar) => { // IN THIS ARM, WE HAVE: @@ -792,6 +795,8 @@ pub(crate) fn codegen_drop<'tcx>( let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); let sig = fx.bcx.import_signature(sig); fx.bcx.ins().call_indirect(sig, drop_fn, &[data]); + // FIXME implement cleanup on exceptions + fx.bcx.ins().jump(ret_block, &[]); } _ => { assert!(!matches!(drop_instance.def, InstanceKind::Virtual(_, _))); @@ -817,10 +822,37 @@ pub(crate) fn codegen_drop<'tcx>( let func_ref = fx.get_function_ref(drop_instance); fx.bcx.ins().call(func_ref, &call_args); + // FIXME implement cleanup on exceptions + fx.bcx.ins().jump(ret_block, &[]); } } } +} - let target_block = fx.get_block(target); - fx.bcx.ins().jump(target_block, &[]); +pub(crate) fn lib_call_arg_param(tcx: TyCtxt<'_>, ty: Type, is_signed: bool) -> AbiParam { + let param = AbiParam::new(ty); + if ty.is_int() && u64::from(ty.bits()) < tcx.data_layout.pointer_size.bits() { + match (&*tcx.sess.target.arch, &*tcx.sess.target.vendor) { + ("x86_64", _) | ("aarch64", "apple") => match (ty, is_signed) { + (types::I8 | types::I16, true) => param.sext(), + (types::I8 | types::I16, false) => param.uext(), + _ => param, + }, + ("aarch64", _) => param, + ("riscv64", _) => match (ty, is_signed) { + (types::I32, _) | (_, true) => param.sext(), + _ => param.uext(), + }, + ("s390x", _) => { + if is_signed { + param.sext() + } else { + param.uext() + } + } + _ => unimplemented!("{:?}", tcx.sess.target.arch), + } + } else { + param + } } diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 06d89bc9ea7d7..6d8614aca6932 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -22,8 +22,10 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam { (RegKind::Integer, 3..=4) => types::I32, (RegKind::Integer, 5..=8) => types::I64, (RegKind::Integer, 9..=16) => types::I128, + (RegKind::Float, 2) => types::F16, (RegKind::Float, 4) => types::F32, (RegKind::Float, 8) => types::F64, + (RegKind::Float, 16) => types::F128, (RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(), _ => unreachable!("{:?}", reg), }; diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 9e048c7badb84..36087f96dd776 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -46,7 +46,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, ret_place: CPlace<'tcx>, - f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option) -> Inst, + f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option) -> SmallVec<[Value; 2]>, ) { let (ret_temp_place, return_ptr) = match ret_arg_abi.mode { PassMode::Ignore => (None, None), @@ -67,23 +67,21 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None), }; - let call_inst = f(fx, return_ptr); + let results = f(fx, return_ptr); match ret_arg_abi.mode { PassMode::Ignore => {} PassMode::Direct(_) => { - let ret_val = fx.bcx.inst_results(call_inst)[0]; + let ret_val = results[0]; ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout)); } PassMode::Pair(_, _) => { - let ret_val_a = fx.bcx.inst_results(call_inst)[0]; - let ret_val_b = fx.bcx.inst_results(call_inst)[1]; + let ret_val_a = results[0]; + let ret_val_b = results[1]; ret_place .write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout)); } PassMode::Cast { ref cast, .. } => { - let results = - fx.bcx.inst_results(call_inst).iter().copied().collect::>(); let result = super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast); ret_place.write_cvalue(fx, result); diff --git a/src/base.rs b/src/base.rs index 524e0d9fe35ed..4617304105a5b 100644 --- a/src/base.rs +++ b/src/base.rs @@ -15,9 +15,9 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use crate::constant::ConstantCx; use crate::debuginfo::{FunctionDebugContext, TypeDebugContext}; -use crate::enable_verifier; use crate::prelude::*; use crate::pretty_clif::CommentWriter; +use crate::{codegen_f16_f128, enable_verifier}; pub(crate) struct CodegenedFunction { symbol_name: String, @@ -193,6 +193,18 @@ pub(crate) fn compile_fn( name = codegened_func.symbol_name )); } + Err(ModuleError::Compilation(CodegenError::Verifier(err))) => { + let early_dcx = rustc_session::EarlyDiagCtxt::new( + rustc_session::config::ErrorOutputType::default(), + ); + let _ = early_dcx.early_err(format!("{:?}", err)); + let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( + &context.func, + Some(Box::new(&clif_comments)), + err, + ); + early_dcx.early_fatal(format!("cranelift verify error:\n{}", pretty_error)); + } Err(err) => { panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name); } @@ -303,7 +315,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap(); }); let inst = fx.bcx.func.layout.last_inst(block).unwrap(); - fx.add_comment(inst, terminator_head); + fx.add_post_comment(inst, terminator_head); } let source_info = bb_data.terminator().source_info; @@ -337,7 +349,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { TerminatorKind::Return => { crate::abi::codegen_return(fx); } - TerminatorKind::Assert { cond, expected, msg, target, unwind: _ } => { + TerminatorKind::Assert { cond, expected, msg, target, unwind } => { if !fx.tcx.sess.overflow_checks() && msg.is_optional_overflow_check() { let target = fx.get_block(*target); fx.bcx.ins().jump(target, &[]); @@ -367,6 +379,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { fx, rustc_hir::LangItem::PanicBoundsCheck, &[index, len, location], + *unwind, Some(source_info.span), ); } @@ -379,6 +392,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { fx, rustc_hir::LangItem::PanicMisalignedPointerDereference, &[required, found, location], + *unwind, Some(source_info.span), ); } @@ -389,6 +403,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { fx, rustc_hir::LangItem::PanicNullPointerDereference, &[location], + *unwind, Some(source_info.span), ) } @@ -399,6 +414,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { fx, msg.panic_function(), &[location], + *unwind, Some(source_info.span), ); } @@ -457,7 +473,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { destination, target, fn_span, - unwind: _, + unwind, call_source: _, } => { fx.tcx.prof.generic_activity("codegen call").run(|| { @@ -468,6 +484,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { args, *destination, *target, + *unwind, ) }); } @@ -514,7 +531,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { ); } TerminatorKind::UnwindTerminate(reason) => { - codegen_unwind_terminate(fx, source_info, *reason); + codegen_unwind_terminate(fx, Some(source_info.span), *reason); } TerminatorKind::UnwindResume => { // FIXME implement unwinding @@ -530,23 +547,19 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { | TerminatorKind::CoroutineDrop => { bug!("shouldn't exist at codegen {:?}", bb_data.terminator()); } - TerminatorKind::Drop { place, target, unwind: _, replace: _, drop, async_fut } => { + TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut } => { assert!( async_fut.is_none() && drop.is_none(), "Async Drop must be expanded or reset to sync before codegen" ); let drop_place = codegen_place(fx, *place); - crate::abi::codegen_drop(fx, source_info, drop_place, *target); + crate::abi::codegen_drop(fx, source_info, drop_place, *target, *unwind); } }; } } -fn codegen_stmt<'tcx>( - fx: &mut FunctionCx<'_, '_, 'tcx>, - #[allow(unused_variables)] cur_block: Block, - stmt: &Statement<'tcx>, -) { +fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: &Statement<'tcx>) { let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt)); fx.set_debug_loc(stmt.source_info); @@ -557,7 +570,7 @@ fn codegen_stmt<'tcx>( if fx.clif_comments.enabled() { let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); with_no_trimmed_paths!({ - fx.add_comment(inst, format!("{:?}", stmt)); + fx.add_post_comment(inst, format!("{:?}", stmt)); }); } } @@ -622,6 +635,15 @@ fn codegen_stmt<'tcx>( let val = operand.load_scalar(fx); match layout.ty.kind() { ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout), + // FIXME(bytecodealliance/wasmtime#8312): Remove + // once backend lowerings have been added to + // Cranelift. + ty::Float(FloatTy::F16) => { + CValue::by_val(codegen_f16_f128::neg_f16(fx, val), layout) + } + ty::Float(FloatTy::F128) => { + CValue::by_val(codegen_f16_f128::neg_f128(fx, val), layout) + } ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout), _ => unreachable!("un op Neg for {:?}", layout.ty), } @@ -793,7 +815,7 @@ fn codegen_stmt<'tcx>( let done_block = fx.bcx.create_block(); let index = fx.bcx.append_block_param(loop_block, fx.pointer_type); let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); - fx.bcx.ins().jump(loop_block, &[zero]); + fx.bcx.ins().jump(loop_block, &[zero.into()]); fx.bcx.switch_to_block(loop_block); let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64); @@ -803,7 +825,7 @@ fn codegen_stmt<'tcx>( let to = lval.place_index(fx, index); to.write_cvalue(fx, operand); let index = fx.bcx.ins().iadd_imm(index, 1); - fx.bcx.ins().jump(loop_block, &[index]); + fx.bcx.ins().jump(loop_block, &[index.into()]); fx.bcx.switch_to_block(done_block); fx.bcx.ins().nop(); @@ -1058,23 +1080,28 @@ pub(crate) fn codegen_panic_nounwind<'tcx>( let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); let args = [msg_ptr, msg_len]; - codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, span); + codegen_panic_inner( + fx, + rustc_hir::LangItem::PanicNounwind, + &args, + UnwindAction::Terminate(UnwindTerminateReason::Abi), + span, + ); } pub(crate) fn codegen_unwind_terminate<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, - source_info: mir::SourceInfo, + span: Option, reason: UnwindTerminateReason, ) { - let args = []; - - codegen_panic_inner(fx, reason.lang_item(), &args, Some(source_info.span)); + codegen_panic_inner(fx, reason.lang_item(), &[], UnwindAction::Unreachable, span); } fn codegen_panic_inner<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, lang_item: rustc_hir::LangItem, args: &[Value], + _unwind: UnwindAction, span: Option, ) { fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); @@ -1090,6 +1117,8 @@ fn codegen_panic_inner<'tcx>( let symbol_name = fx.tcx.symbol_name(instance).name; + // FIXME implement cleanup on exceptions + fx.lib_call( symbol_name, args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(), diff --git a/src/cast.rs b/src/cast.rs index e2346324232fc..8a725680e7059 100644 --- a/src/cast.rs +++ b/src/cast.rs @@ -1,5 +1,6 @@ //! Various number casting functions +use crate::codegen_f16_f128; use crate::prelude::*; pub(crate) fn clif_intcast( @@ -36,6 +37,14 @@ pub(crate) fn clif_int_or_float_cast( ) -> Value { let from_ty = fx.bcx.func.dfg.value_type(from); + // FIXME(bytecodealliance/wasmtime#8312): Remove in favour of native + // Cranelift operations once Cranelift backends have lowerings for them. + if matches!(from_ty, types::F16 | types::F128) + || matches!(to_ty, types::F16 | types::F128) && from_ty != to_ty + { + return codegen_f16_f128::codegen_cast(fx, from, from_signed, to_ty, to_signed); + } + if from_ty.is_int() && to_ty.is_int() { // int-like -> int-like clif_intcast( @@ -58,8 +67,10 @@ pub(crate) fn clif_int_or_float_cast( "__float{sign}ti{flt}f", sign = if from_signed { "" } else { "un" }, flt = match to_ty { + types::F16 => "h", types::F32 => "s", types::F64 => "d", + types::F128 => "t", _ => unreachable!("{:?}", to_ty), }, ); @@ -90,8 +101,10 @@ pub(crate) fn clif_int_or_float_cast( "__fix{sign}{flt}fti", sign = if to_signed { "" } else { "uns" }, flt = match from_ty { + types::F16 => "h", types::F32 => "s", types::F64 => "d", + types::F128 => "t", _ => unreachable!("{:?}", to_ty), }, ); @@ -145,8 +158,12 @@ pub(crate) fn clif_int_or_float_cast( } else if from_ty.is_float() && to_ty.is_float() { // float -> float match (from_ty, to_ty) { - (types::F32, types::F64) => fx.bcx.ins().fpromote(types::F64, from), - (types::F64, types::F32) => fx.bcx.ins().fdemote(types::F32, from), + (types::F16, types::F32 | types::F64 | types::F128) + | (types::F32, types::F64 | types::F128) + | (types::F64, types::F128) => fx.bcx.ins().fpromote(to_ty, from), + (types::F128, types::F64 | types::F32 | types::F16) + | (types::F64, types::F32 | types::F16) + | (types::F32, types::F16) => fx.bcx.ins().fdemote(to_ty, from), _ => from, } } else { diff --git a/src/codegen_f16_f128.rs b/src/codegen_f16_f128.rs new file mode 100644 index 0000000000000..1e202be1f1855 --- /dev/null +++ b/src/codegen_f16_f128.rs @@ -0,0 +1,284 @@ +use crate::prelude::*; + +pub(crate) fn f16_to_f32(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let (value, arg_ty) = + if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { + ( + fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value), + lib_call_arg_param(fx.tcx, types::I16, false), + ) + } else { + (value, AbiParam::new(types::F16)) + }; + fx.lib_call("__extendhfsf2", vec![arg_ty], vec![AbiParam::new(types::F32)], &[value])[0] +} + +fn f16_to_f64(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let ret = f16_to_f32(fx, value); + fx.bcx.ins().fpromote(types::F64, ret) +} + +pub(crate) fn f32_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { + types::I16 + } else { + types::F16 + }; + let ret = fx.lib_call( + "__truncsfhf2", + vec![AbiParam::new(types::F32)], + vec![AbiParam::new(ret_ty)], + &[value], + )[0]; + if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret } +} + +fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { + types::I16 + } else { + types::F16 + }; + let ret = fx.lib_call( + "__truncdfhf2", + vec![AbiParam::new(types::F64)], + vec![AbiParam::new(ret_ty)], + &[value], + )[0]; + if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret } +} + +pub(crate) fn fcmp(fx: &mut FunctionCx<'_, '_, '_>, cc: FloatCC, lhs: Value, rhs: Value) -> Value { + let ty = fx.bcx.func.dfg.value_type(lhs); + match ty { + types::F32 | types::F64 => fx.bcx.ins().fcmp(cc, lhs, rhs), + types::F16 => { + let lhs = f16_to_f32(fx, lhs); + let rhs = f16_to_f32(fx, rhs); + fx.bcx.ins().fcmp(cc, lhs, rhs) + } + types::F128 => { + let (name, int_cc) = match cc { + FloatCC::Equal => ("__eqtf2", IntCC::Equal), + FloatCC::NotEqual => ("__netf2", IntCC::NotEqual), + FloatCC::LessThan => ("__lttf2", IntCC::SignedLessThan), + FloatCC::LessThanOrEqual => ("__letf2", IntCC::SignedLessThanOrEqual), + FloatCC::GreaterThan => ("__gttf2", IntCC::SignedGreaterThan), + FloatCC::GreaterThanOrEqual => ("__getf2", IntCC::SignedGreaterThanOrEqual), + _ => unreachable!("not currently used in rustc_codegen_cranelift: {cc:?}"), + }; + let res = fx.lib_call( + name, + vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], + // FIXME(rust-lang/compiler-builtins#919): This should be `I64` on non-AArch64 + // architectures, but switching it before compiler-builtins is fixed causes test + // failures. + vec![AbiParam::new(types::I32)], + &[lhs, rhs], + )[0]; + let zero = fx.bcx.ins().iconst(types::I32, 0); + let res = fx.bcx.ins().icmp(int_cc, res, zero); + res + } + _ => unreachable!("{ty:?}"), + } +} + +pub(crate) fn codegen_f128_binop( + fx: &mut FunctionCx<'_, '_, '_>, + bin_op: BinOp, + lhs: Value, + rhs: Value, +) -> Value { + let name = match bin_op { + BinOp::Add => "__addtf3", + BinOp::Sub => "__subtf3", + BinOp::Mul => "__multf3", + BinOp::Div => "__divtf3", + _ => unreachable!("handled in `codegen_float_binop`"), + }; + fx.lib_call( + name, + vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], + vec![AbiParam::new(types::F128)], + &[lhs, rhs], + )[0] +} + +pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value); + let bits = fx.bcx.ins().bxor_imm(bits, 0x8000); + fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits) +} + +pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value); + let (low, high) = fx.bcx.ins().isplit(bits); + let high = fx.bcx.ins().bxor_imm(high, 0x8000_0000_0000_0000_u64 as i64); + let bits = fx.bcx.ins().iconcat(low, high); + fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits) +} + +pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value); + let bits = fx.bcx.ins().band_imm(bits, 0x7fff); + fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits) +} + +pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value); + let (low, high) = fx.bcx.ins().isplit(bits); + let high = fx.bcx.ins().band_imm(high, 0x7fff_ffff_ffff_ffff_u64 as i64); + let bits = fx.bcx.ins().iconcat(low, high); + fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits) +} + +pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { + let lhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), lhs); + let rhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), rhs); + let res = fx.bcx.ins().band_imm(lhs, 0x7fff); + let sign = fx.bcx.ins().band_imm(rhs, 0x8000); + let res = fx.bcx.ins().bor(res, sign); + fx.bcx.ins().bitcast(types::F16, MemFlags::new(), res) +} + +pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { + let lhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), lhs); + let rhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), rhs); + let (low, lhs_high) = fx.bcx.ins().isplit(lhs); + let (_, rhs_high) = fx.bcx.ins().isplit(rhs); + let high = fx.bcx.ins().band_imm(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64); + let sign = fx.bcx.ins().band_imm(rhs_high, 0x8000_0000_0000_0000_u64 as i64); + let high = fx.bcx.ins().bor(high, sign); + let res = fx.bcx.ins().iconcat(low, high); + fx.bcx.ins().bitcast(types::F128, MemFlags::new(), res) +} + +pub(crate) fn codegen_cast( + fx: &mut FunctionCx<'_, '_, '_>, + from: Value, + from_signed: bool, + to_ty: Type, + to_signed: bool, +) -> Value { + let from_ty = fx.bcx.func.dfg.value_type(from); + if from_ty.is_float() && to_ty.is_float() { + let name = match (from_ty, to_ty) { + (types::F16, types::F32) => return f16_to_f32(fx, from), + (types::F16, types::F64) => return f16_to_f64(fx, from), + (types::F16, types::F128) => "__extendhftf2", + (types::F32, types::F128) => "__extendsftf2", + (types::F64, types::F128) => "__extenddftf2", + (types::F128, types::F64) => "__trunctfdf2", + (types::F128, types::F32) => "__trunctfsf2", + (types::F128, types::F16) => "__trunctfhf2", + (types::F64, types::F16) => return f64_to_f16(fx, from), + (types::F32, types::F16) => return f32_to_f16(fx, from), + _ => unreachable!("{from_ty:?} -> {to_ty:?}"), + }; + fx.lib_call(name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])[0] + } else if from_ty.is_int() && to_ty == types::F16 { + let res = clif_int_or_float_cast(fx, from, from_signed, types::F32, false); + f32_to_f16(fx, res) + } else if from_ty == types::F16 && to_ty.is_int() { + let from = f16_to_f32(fx, from); + clif_int_or_float_cast(fx, from, false, to_ty, to_signed) + } else if from_ty.is_int() && to_ty == types::F128 { + let (from, from_ty) = if from_ty.bits() < 32 { + (clif_int_or_float_cast(fx, from, from_signed, types::I32, from_signed), types::I32) + } else { + (from, from_ty) + }; + let name = format!( + "__float{sign}{size}itf", + sign = if from_signed { "" } else { "un" }, + size = match from_ty { + types::I32 => 's', + types::I64 => 'd', + types::I128 => 't', + _ => unreachable!("{from_ty:?}"), + }, + ); + fx.lib_call( + &name, + vec![lib_call_arg_param(fx.tcx, from_ty, from_signed)], + vec![AbiParam::new(to_ty)], + &[from], + )[0] + } else if from_ty == types::F128 && to_ty.is_int() { + let ret_ty = if to_ty.bits() < 32 { types::I32 } else { to_ty }; + let name = format!( + "__fix{sign}tf{size}i", + sign = if from_signed { "" } else { "un" }, + size = match ret_ty { + types::I32 => 's', + types::I64 => 'd', + types::I128 => 't', + _ => unreachable!("{from_ty:?}"), + }, + ); + let ret = + fx.lib_call(&name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from]) + [0]; + let val = if ret_ty == to_ty { + ret + } else { + let (min, max) = match (to_ty, to_signed) { + (types::I8, false) => (0, i64::from(u8::MAX)), + (types::I16, false) => (0, i64::from(u16::MAX)), + (types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)), + (types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)), + _ => unreachable!("{to_ty:?}"), + }; + let min_val = fx.bcx.ins().iconst(types::I32, min); + let max_val = fx.bcx.ins().iconst(types::I32, max); + + let val = if to_signed { + let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, ret, min); + let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, ret, max); + let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, ret); + fx.bcx.ins().select(has_overflow, max_val, bottom_capped) + } else { + let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, ret, max); + fx.bcx.ins().select(has_overflow, max_val, ret) + }; + fx.bcx.ins().ireduce(to_ty, val) + }; + + if let Some(false) = fx.tcx.sess.opts.unstable_opts.saturating_float_casts { + return val; + } + + let is_not_nan = fcmp(fx, FloatCC::Equal, from, from); + let zero = type_zero_value(&mut fx.bcx, to_ty); + fx.bcx.ins().select(is_not_nan, val, zero) + } else { + unreachable!("{from_ty:?} -> {to_ty:?}"); + } +} + +pub(crate) fn fma_f16(fx: &mut FunctionCx<'_, '_, '_>, x: Value, y: Value, z: Value) -> Value { + let x = f16_to_f64(fx, x); + let y = f16_to_f64(fx, y); + let z = f16_to_f64(fx, z); + let res = fx.bcx.ins().fma(x, y, z); + f64_to_f16(fx, res) +} + +pub(crate) fn fmin_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { + fx.lib_call( + "fminimumf128", + vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], + vec![AbiParam::new(types::F128)], + &[a, b], + )[0] +} + +pub(crate) fn fmax_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { + fx.lib_call( + "fmaximumf128", + vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], + vec![AbiParam::new(types::F128)], + &[a, b], + )[0] +} diff --git a/src/common.rs b/src/common.rs index abe2972ba0cbd..2f11b2d2dcc1a 100644 --- a/src/common.rs +++ b/src/common.rs @@ -33,10 +33,10 @@ pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type { Integer::I128 => types::I128, }, Primitive::Float(float) => match float { - Float::F16 => unimplemented!("f16_f128"), + Float::F16 => types::F16, Float::F32 => types::F32, Float::F64 => types::F64, - Float::F128 => unimplemented!("f16_f128"), + Float::F128 => types::F128, }, // FIXME(erikdesjardins): handle non-default addrspace ptr sizes Primitive::Pointer(_) => pointer_ty(tcx), @@ -64,10 +64,10 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option types::I32, ty::Float(size) => match size { - FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F16 => types::F16, FloatTy::F32 => types::F32, FloatTy::F64 => types::F64, - FloatTy::F128 => unimplemented!("f16_f128"), + FloatTy::F128 => types::F128, }, ty::FnPtr(..) => pointer_ty(tcx), ty::RawPtr(pointee_ty, _) | ty::Ref(_, pointee_ty, _) => { diff --git a/src/compiler_builtins.rs b/src/compiler_builtins.rs index bf16e81a06f9f..6eea19211fa1b 100644 --- a/src/compiler_builtins.rs +++ b/src/compiler_builtins.rs @@ -46,15 +46,100 @@ builtin_functions! { fn __rust_u128_mulo(a: u128, b: u128, oflow: &mut i32) -> u128; fn __rust_i128_mulo(a: i128, b: i128, oflow: &mut i32) -> i128; - // floats + // integer -> float fn __floattisf(i: i128) -> f32; fn __floattidf(i: i128) -> f64; + fn __floatsitf(i: i32) -> f128; + fn __floatditf(i: i64) -> f128; + fn __floattitf(i: i128) -> f128; fn __floatuntisf(i: u128) -> f32; fn __floatuntidf(i: u128) -> f64; + fn __floatunsitf(i: u32) -> f128; + fn __floatunditf(i: u64) -> f128; + fn __floatuntitf(i: u128) -> f128; + // float -> integer fn __fixsfti(f: f32) -> i128; fn __fixdfti(f: f64) -> i128; + fn __fixtfsi(f: f128) -> i32; + fn __fixtfdi(f: f128) -> i64; + fn __fixtfti(f: f128) -> i128; fn __fixunssfti(f: f32) -> u128; fn __fixunsdfti(f: f64) -> u128; + fn __fixunstfsi(f: f128) -> u32; + fn __fixunstfdi(f: f128) -> u64; + fn __fixunstfti(f: f128) -> u128; + // float -> float + fn __extendhfsf2(f: f16) -> f32; + fn __extendhftf2(f: f16) -> f128; + fn __extendsftf2(f: f32) -> f128; + fn __extenddftf2(f: f64) -> f128; + fn __trunctfdf2(f: f128) -> f64; + fn __trunctfsf2(f: f128) -> f32; + fn __trunctfhf2(f: f128) -> f16; + fn __truncdfhf2(f: f64) -> f16; + fn __truncsfhf2(f: f32) -> f16; + // float binops + fn __addtf3(a: f128, b: f128) -> f128; + fn __subtf3(a: f128, b: f128) -> f128; + fn __multf3(a: f128, b: f128) -> f128; + fn __divtf3(a: f128, b: f128) -> f128; + fn fmodf(a: f32, b: f32) -> f32; + fn fmod(a: f64, b: f64) -> f64; + fn fmodf128(a: f128, b: f128) -> f128; + // float comparison + fn __eqtf2(a: f128, b: f128) -> i32; + fn __netf2(a: f128, b: f128) -> i32; + fn __lttf2(a: f128, b: f128) -> i32; + fn __letf2(a: f128, b: f128) -> i32; + fn __gttf2(a: f128, b: f128) -> i32; + fn __getf2(a: f128, b: f128) -> i32; + fn fminimumf128(a: f128, b: f128) -> f128; + fn fmaximumf128(a: f128, b: f128) -> f128; + // Cranelift float libcalls + fn fmaf(a: f32, b: f32, c: f32) -> f32; + fn fma(a: f64, b: f64, c: f64) -> f64; + fn floorf(f: f32) -> f32; + fn floor(f: f64) -> f64; + fn ceilf(f: f32) -> f32; + fn ceil(f: f64) -> f64; + fn truncf(f: f32) -> f32; + fn trunc(f: f64) -> f64; + fn nearbyintf(f: f32) -> f32; + fn nearbyint(f: f64) -> f64; + // float intrinsics + fn __powisf2(a: f32, b: i32) -> f32; + fn __powidf2(a: f64, b: i32) -> f64; + // FIXME(f16_f128): `compiler-builtins` doesn't currently support `__powitf2` on MSVC. + // fn __powitf2(a: f128, b: i32) -> f128; + fn powf(a: f32, b: f32) -> f32; + fn pow(a: f64, b: f64) -> f64; + fn expf(f: f32) -> f32; + fn exp(f: f64) -> f64; + fn exp2f(f: f32) -> f32; + fn exp2(f: f64) -> f64; + fn logf(f: f32) -> f32; + fn log(f: f64) -> f64; + fn log2f(f: f32) -> f32; + fn log2(f: f64) -> f64; + fn log10f(f: f32) -> f32; + fn log10(f: f64) -> f64; + fn sinf(f: f32) -> f32; + fn sin(f: f64) -> f64; + fn cosf(f: f32) -> f32; + fn cos(f: f64) -> f64; + fn fmaf128(a: f128, b: f128, c: f128) -> f128; + fn floorf16(f: f16) -> f16; + fn floorf128(f: f128) -> f128; + fn ceilf16(f: f16) -> f16; + fn ceilf128(f: f128) -> f128; + fn truncf16(f: f16) -> f16; + fn truncf128(f: f128) -> f128; + fn rintf16(f: f16) -> f16; + fn rintf128(f: f128) -> f128; + fn sqrtf16(f: f16) -> f16; + fn sqrtf128(f: f128) -> f128; + // FIXME(f16_f128): Add other float intrinsics as compiler-builtins gains support (meaning they + // are available on all targets). // allocator // NOTE: These need to be mentioned here despite not being part of compiler_builtins because @@ -67,5 +152,4 @@ builtin_functions! { fn malloc(size: size_t) -> *mut c_void; fn realloc(p: *mut c_void, size: size_t) -> *mut c_void; fn free(p: *mut c_void) -> (); - } diff --git a/src/debuginfo/emit.rs b/src/debuginfo/emit.rs index ccdc347af660f..0f4696b9337e7 100644 --- a/src/debuginfo/emit.rs +++ b/src/debuginfo/emit.rs @@ -81,13 +81,36 @@ impl WriterRelocate { /// Perform the collected relocations to be usable for JIT usage. #[cfg(all(feature = "jit", not(windows)))] pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec { + use cranelift_module::Module; + for reloc in self.relocs.drain(..) { match reloc.name { super::DebugRelocName::Section(_) => unreachable!(), super::DebugRelocName::Symbol(sym) => { - let addr = jit_module.get_finalized_function( - cranelift_module::FuncId::from_u32(sym.try_into().unwrap()), - ); + let addr = if sym & 1 << 31 == 0 { + let func_id = FuncId::from_u32(sym.try_into().unwrap()); + // FIXME make JITModule::get_address public and use it here instead. + // HACK rust_eh_personality is likely not defined in the same crate, + // so get_finalized_function won't work. Use the rust_eh_personality + // of cg_clif itself, which is likely ABI compatible. + if jit_module.declarations().get_function_decl(func_id).name.as_deref() + == Some("rust_eh_personality") + { + extern "C" { + fn rust_eh_personality() -> !; + } + rust_eh_personality as *const u8 + } else { + jit_module.get_finalized_function(func_id) + } + } else { + jit_module + .get_finalized_data(DataId::from_u32( + u32::try_from(sym).unwrap() & !(1 << 31), + )) + .0 + }; + let val = (addr as u64 as i64 + reloc.addend) as u64; self.writer.write_udata_at(reloc.offset as usize, val, reloc.size).unwrap(); } @@ -196,6 +219,16 @@ impl Writer for WriterRelocate { }); self.write_udata(0, size) } + gimli::DW_EH_PE_absptr => { + self.relocs.push(DebugReloc { + offset: self.len() as u32, + size: size.into(), + name: DebugRelocName::Symbol(symbol), + addend, + kind: object::RelocationKind::Absolute, + }); + self.write_udata(0, size.into()) + } _ => Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)), }, } diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index 362333d35a411..74b82a7139ab0 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -1,7 +1,6 @@ //! Unwind info generation (`.eh_frame`) use cranelift_codegen::ir::Endianness; -use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::isa::unwind::UnwindInfo; use cranelift_object::ObjectProduct; use gimli::RunTimeEndian; @@ -18,14 +17,14 @@ pub(crate) struct UnwindContext { } impl UnwindContext { - pub(crate) fn new(isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { - let endian = match isa.endianness() { + pub(crate) fn new(module: &mut dyn Module, pic_eh_frame: bool) -> Self { + let endian = match module.isa().endianness() { Endianness::Little => RunTimeEndian::Little, Endianness::Big => RunTimeEndian::Big, }; let mut frame_table = FrameTable::default(); - let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { + let cie_id = if let Some(mut cie) = module.isa().create_systemv_cie() { if pic_eh_frame { cie.fde_address_encoding = gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0); @@ -38,8 +37,15 @@ impl UnwindContext { UnwindContext { endian, frame_table, cie_id } } - pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) { - if let target_lexicon::OperatingSystem::MacOSX { .. } = isa.triple().operating_system { + pub(crate) fn add_function( + &mut self, + module: &mut dyn Module, + func_id: FuncId, + context: &Context, + ) { + if let target_lexicon::OperatingSystem::MacOSX { .. } = + module.isa().triple().operating_system + { // The object crate doesn't currently support DW_GNU_EH_PE_absptr, which macOS // requires for unwinding tables. In addition on arm64 it currently doesn't // support 32bit relocations as we currently use for the unwinding table. @@ -48,7 +54,7 @@ impl UnwindContext { } let unwind_info = if let Some(unwind_info) = - context.compiled_code().unwrap().create_unwind_info(isa).unwrap() + context.compiled_code().unwrap().create_unwind_info(module.isa()).unwrap() { unwind_info } else { diff --git a/src/driver/jit.rs b/src/driver/jit.rs index e368cf4386d01..b1f185b551c3c 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -6,6 +6,7 @@ use std::os::raw::{c_char, c_int}; use cranelift_jit::{JITBuilder, JITModule}; use rustc_codegen_ssa::CrateInfo; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::mono::MonoItem; use rustc_session::Session; use rustc_span::sym; @@ -84,7 +85,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec) -> ! { tcx.dcx().abort_if_errors(); - jit_module.finalize_definitions(); + let mut jit_module = jit_module.finalize_definitions(); println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" @@ -104,7 +105,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec) -> ! { call_conv: jit_module.target_config().default_call_conv, }; let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); - let finalized_start: *const u8 = jit_module.module.get_finalized_function(start_func_id); + let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); let f: extern "C" fn(c_int, *const *const c_char) -> c_int = unsafe { ::std::mem::transmute(finalized_start) }; @@ -119,7 +120,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec) -> ! { std::process::exit(ret); } -pub(crate) fn codegen_and_compile_fn<'tcx>( +fn codegen_and_compile_fn<'tcx>( tcx: TyCtxt<'tcx>, cx: &mut crate::CodegenCx, cached_context: &mut Context, diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index eb0dfbb69c3b7..2e02e85a997d5 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -66,7 +66,12 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( fx.tcx .dcx() .warn(format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic)); - crate::trap::trap_unimplemented(fx, intrinsic); + let msg = format!( + "{intrinsic} is not yet supported.\n\ + See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\ + Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues" + ); + crate::base::codegen_panic_nounwind(fx, &msg, None); return; } } diff --git a/src/intrinsics/llvm_aarch64.rs b/src/intrinsics/llvm_aarch64.rs index 387c87d123a39..d22483cf17764 100644 --- a/src/intrinsics/llvm_aarch64.rs +++ b/src/intrinsics/llvm_aarch64.rs @@ -7,7 +7,7 @@ use crate::inline_asm::{CInlineAsmOperand, codegen_inline_asm_inner}; use crate::intrinsics::*; use crate::prelude::*; -pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( +pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, intrinsic: &str, args: &[Spanned>], @@ -507,7 +507,12 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( "unsupported AArch64 llvm intrinsic {}; replacing with trap", intrinsic )); - crate::trap::trap_unimplemented(fx, intrinsic); + let msg = format!( + "{intrinsic} is not yet supported.\n\ + See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\ + Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues" + ); + crate::base::codegen_panic_nounwind(fx, &msg, None); return; } } diff --git a/src/intrinsics/llvm_x86.rs b/src/intrinsics/llvm_x86.rs index c02d31844e032..3d67913a8fff1 100644 --- a/src/intrinsics/llvm_x86.rs +++ b/src/intrinsics/llvm_x86.rs @@ -7,7 +7,7 @@ use crate::inline_asm::{CInlineAsmOperand, codegen_inline_asm_inner}; use crate::intrinsics::*; use crate::prelude::*; -pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( +pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, intrinsic: &str, args: &[Spanned>], @@ -147,10 +147,10 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( let offset = fx.bcx.ins().imul(index_lane, scale); let lane_ptr = fx.bcx.ins().iadd(ptr, offset); let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), lane_ptr, 0); - fx.bcx.ins().jump(next, &[res]); + fx.bcx.ins().jump(next, &[res.into()]); fx.bcx.switch_to_block(if_disabled); - fx.bcx.ins().jump(next, &[src_lane]); + fx.bcx.ins().jump(next, &[src_lane.into()]); fx.bcx.seal_block(next); fx.bcx.switch_to_block(next); @@ -1316,7 +1316,12 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( fx.tcx .dcx() .warn(format!("unsupported x86 llvm intrinsic {}; replacing with trap", intrinsic)); - crate::trap::trap_unimplemented(fx, intrinsic); + let msg = format!( + "{intrinsic} is not yet supported.\n\ + See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\ + Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues" + ); + crate::base::codegen_panic_nounwind(fx, &msg, None); return; } } diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 9018d78b00aeb..b21ca32c9a2e3 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -27,6 +27,7 @@ use rustc_span::{Symbol, sym}; pub(crate) use self::llvm::codegen_llvm_intrinsic_call; use crate::cast::clif_intcast; +use crate::codegen_f16_f128; use crate::prelude::*; fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! { @@ -248,8 +249,10 @@ fn bool_to_zero_or_max_uint<'tcx>( let ty = fx.clif_type(ty).unwrap(); let int_ty = match ty { + types::F16 => types::I16, types::F32 => types::I32, types::F64 => types::I64, + types::F128 => types::I128, ty => ty, }; @@ -308,45 +311,83 @@ fn codegen_float_intrinsic_call<'tcx>( ret: CPlace<'tcx>, ) -> bool { let (name, arg_count, ty, clif_ty) = match intrinsic { + sym::expf16 => ("expf16", 1, fx.tcx.types.f16, types::F16), sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32), sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64), + sym::expf128 => ("expf128", 1, fx.tcx.types.f128, types::F128), + sym::exp2f16 => ("exp2f16", 1, fx.tcx.types.f16, types::F16), sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32), sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64), + sym::exp2f128 => ("exp2f128", 1, fx.tcx.types.f128, types::F128), + sym::sqrtf16 => ("sqrtf16", 1, fx.tcx.types.f16, types::F16), sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32), sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64), + sym::sqrtf128 => ("sqrtf128", 1, fx.tcx.types.f128, types::F128), + sym::powif16 => ("__powisf2", 2, fx.tcx.types.f16, types::F16), // compiler-builtins sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins + sym::powif128 => ("__powitf2", 2, fx.tcx.types.f128, types::F128), // compiler-builtins + sym::powf16 => ("powf16", 2, fx.tcx.types.f16, types::F16), sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32), sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64), + sym::powf128 => ("powf128", 2, fx.tcx.types.f128, types::F128), + sym::logf16 => ("logf16", 1, fx.tcx.types.f16, types::F16), sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32), sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64), + sym::logf128 => ("logf128", 1, fx.tcx.types.f128, types::F128), + sym::log2f16 => ("log2f16", 1, fx.tcx.types.f16, types::F16), sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32), sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64), + sym::log2f128 => ("log2f128", 1, fx.tcx.types.f128, types::F128), + sym::log10f16 => ("log10f16", 1, fx.tcx.types.f16, types::F16), sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32), sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64), + sym::log10f128 => ("log10f128", 1, fx.tcx.types.f128, types::F128), + sym::fabsf16 => ("fabsf16", 1, fx.tcx.types.f16, types::F16), sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32), sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64), + sym::fabsf128 => ("fabsf128", 1, fx.tcx.types.f128, types::F128), + sym::fmaf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16), sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64), + sym::fmaf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128), // FIXME: calling `fma` from libc without FMA target feature uses expensive sofware emulation + sym::fmuladdf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f16 sym::fmuladdf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f32 sym::fmuladdf64 => ("fma", 3, fx.tcx.types.f64, types::F64), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f64 + sym::fmuladdf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f128 + sym::copysignf16 => ("copysignf16", 2, fx.tcx.types.f16, types::F16), sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32), sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64), + sym::copysignf128 => ("copysignf128", 2, fx.tcx.types.f128, types::F128), + sym::floorf16 => ("floorf16", 1, fx.tcx.types.f16, types::F16), sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32), sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64), + sym::floorf128 => ("floorf128", 1, fx.tcx.types.f128, types::F128), + sym::ceilf16 => ("ceilf16", 1, fx.tcx.types.f16, types::F16), sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32), sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64), + sym::ceilf128 => ("ceilf128", 1, fx.tcx.types.f128, types::F128), + sym::truncf16 => ("truncf16", 1, fx.tcx.types.f16, types::F16), sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32), sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64), + sym::truncf128 => ("truncf128", 1, fx.tcx.types.f128, types::F128), + sym::round_ties_even_f16 => ("rintf16", 1, fx.tcx.types.f16, types::F16), sym::round_ties_even_f32 => ("rintf", 1, fx.tcx.types.f32, types::F32), sym::round_ties_even_f64 => ("rint", 1, fx.tcx.types.f64, types::F64), + sym::round_ties_even_f128 => ("rintf128", 1, fx.tcx.types.f128, types::F128), + sym::roundf16 => ("roundf16", 1, fx.tcx.types.f16, types::F16), sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32), sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64), + sym::roundf128 => ("roundf128", 1, fx.tcx.types.f128, types::F128), + sym::sinf16 => ("sinf16", 1, fx.tcx.types.f16, types::F16), sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32), sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64), + sym::sinf128 => ("sinf128", 1, fx.tcx.types.f128, types::F128), + sym::cosf16 => ("cosf16", 1, fx.tcx.types.f16, types::F16), sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32), sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64), + sym::cosf128 => ("cosf128", 1, fx.tcx.types.f128, types::F128), _ => return false, }; @@ -379,13 +420,26 @@ fn codegen_float_intrinsic_call<'tcx>( }; let layout = fx.layout_of(ty); + // FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift operations + // for `f16` and `f128` once the lowerings have been implemented in Cranelift. let res = match intrinsic { + sym::fmaf16 | sym::fmuladdf16 => { + CValue::by_val(codegen_f16_f128::fma_f16(fx, args[0], args[1], args[2]), layout) + } sym::fmaf32 | sym::fmaf64 | sym::fmuladdf32 | sym::fmuladdf64 => { CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout) } + sym::copysignf16 => { + CValue::by_val(codegen_f16_f128::copysign_f16(fx, args[0], args[1]), layout) + } + sym::copysignf128 => { + CValue::by_val(codegen_f16_f128::copysign_f128(fx, args[0], args[1]), layout) + } sym::copysignf32 | sym::copysignf64 => { CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout) } + sym::fabsf16 => CValue::by_val(codegen_f16_f128::abs_f16(fx, args[0]), layout), + sym::fabsf128 => CValue::by_val(codegen_f16_f128::abs_f128(fx, args[0]), layout), sym::fabsf32 | sym::fabsf64 | sym::floorf32 @@ -415,11 +469,36 @@ fn codegen_float_intrinsic_call<'tcx>( // These intrinsics aren't supported natively by Cranelift. // Lower them to a libcall. - sym::powif32 | sym::powif64 => { - let input_tys: Vec<_> = vec![AbiParam::new(clif_ty), AbiParam::new(types::I32)]; + sym::powif16 | sym::powif32 | sym::powif64 | sym::powif128 => { + let temp; + let (clif_ty, args) = if intrinsic == sym::powif16 { + temp = [codegen_f16_f128::f16_to_f32(fx, args[0]), args[1]]; + (types::F32, temp.as_slice()) + } else { + (clif_ty, args) + }; + let input_tys: Vec<_> = + vec![AbiParam::new(clif_ty), lib_call_arg_param(fx.tcx, types::I32, true)]; let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0]; + let ret_val = if intrinsic == sym::powif16 { + codegen_f16_f128::f32_to_f16(fx, ret_val) + } else { + ret_val + }; CValue::by_val(ret_val, fx.layout_of(ty)) } + sym::powf16 => { + // FIXME(f16_f128): Rust `compiler-builtins` doesn't export `powf16` yet. + let x = codegen_f16_f128::f16_to_f32(fx, args[0]); + let y = codegen_f16_f128::f16_to_f32(fx, args[1]); + let ret_val = fx.lib_call( + "powf", + vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], + vec![AbiParam::new(types::F32)], + &[x, y], + )[0]; + CValue::by_val(codegen_f16_f128::f32_to_f16(fx, ret_val), fx.layout_of(ty)) + } _ => { let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect(); let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0]; @@ -801,7 +880,11 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME implement 128bit atomics if fx.tcx.is_compiler_builtins(LOCAL_CRATE) { // special case for compiler-builtins to avoid having to patch it - crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported"); + crate::base::codegen_panic_nounwind( + fx, + "128bit atomics not yet supported", + None, + ); return Ok(()); } else { fx.tcx @@ -832,7 +915,11 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME implement 128bit atomics if fx.tcx.is_compiler_builtins(LOCAL_CRATE) { // special case for compiler-builtins to avoid having to patch it - crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported"); + crate::base::codegen_panic_nounwind( + fx, + "128bit atomics not yet supported", + None, + ); return Ok(()); } else { fx.tcx @@ -1109,6 +1196,20 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, old); } + sym::minimumf16 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + // FIXME(bytecodealliance/wasmtime#8312): Use `fmin` directly once + // Cranelift backend lowerings are implemented. + let a = codegen_f16_f128::f16_to_f32(fx, a); + let b = codegen_f16_f128::f16_to_f32(fx, b); + let val = fx.bcx.ins().fmin(a, b); + let val = codegen_f16_f128::f32_to_f16(fx, val); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16)); + ret.write_cvalue(fx, val); + } sym::minimumf32 => { intrinsic_args!(fx, args => (a, b); intrinsic); let a = a.load_scalar(fx); @@ -1127,6 +1228,31 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); ret.write_cvalue(fx, val); } + sym::minimumf128 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + // FIXME(bytecodealliance/wasmtime#8312): Use `fmin` once Cranelift + // backend lowerings are implemented. + let val = codegen_f16_f128::fmin_f128(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128)); + ret.write_cvalue(fx, val); + } + sym::maximumf16 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + // FIXME(bytecodealliance/wasmtime#8312): Use `fmax` directly once + // Cranelift backend lowerings are implemented. + let a = codegen_f16_f128::f16_to_f32(fx, a); + let b = codegen_f16_f128::f16_to_f32(fx, b); + let val = fx.bcx.ins().fmax(a, b); + let val = codegen_f16_f128::f32_to_f16(fx, val); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16)); + ret.write_cvalue(fx, val); + } sym::maximumf32 => { intrinsic_args!(fx, args => (a, b); intrinsic); let a = a.load_scalar(fx); @@ -1145,7 +1271,27 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); ret.write_cvalue(fx, val); } + sym::maximumf128 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + // FIXME(bytecodealliance/wasmtime#8312): Use `fmax` once Cranelift + // backend lowerings are implemented. + let val = codegen_f16_f128::fmax_f128(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128)); + ret.write_cvalue(fx, val); + } + + sym::minnumf16 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let val = crate::num::codegen_float_min(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16)); + ret.write_cvalue(fx, val); + } sym::minnumf32 => { intrinsic_args!(fx, args => (a, b); intrinsic); let a = a.load_scalar(fx); @@ -1164,6 +1310,24 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); ret.write_cvalue(fx, val); } + sym::minnumf128 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let val = crate::num::codegen_float_min(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128)); + ret.write_cvalue(fx, val); + } + sym::maxnumf16 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let val = crate::num::codegen_float_max(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16)); + ret.write_cvalue(fx, val); + } sym::maxnumf32 => { intrinsic_args!(fx, args => (a, b); intrinsic); let a = a.load_scalar(fx); @@ -1182,6 +1346,15 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); ret.write_cvalue(fx, val); } + sym::maxnumf128 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let val = crate::num::codegen_float_max(fx, a, b); + let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128)); + ret.write_cvalue(fx, val); + } sym::catch_unwind => { intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic); diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index dd6d8dbb6f5e0..46a441488fa6a 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -283,6 +283,20 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( ret_lane.write_cvalue(fx, val); } + sym::simd_insert_dyn => { + intrinsic_args!(fx, args => (base, idx, val); intrinsic); + + if !base.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, base.layout().ty); + return; + } + + let idx = idx.load_scalar(fx); + + ret.write_cvalue(fx, base); + ret.write_lane_dyn(fx, idx, val); + } + sym::simd_extract => { let (v, idx) = match args { [v, idx] => (v, idx), @@ -318,6 +332,20 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( ret.write_cvalue(fx, ret_lane); } + sym::simd_extract_dyn => { + intrinsic_args!(fx, args => (v, idx); intrinsic); + + if !v.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); + return; + } + + let idx = idx.load_scalar(fx); + + let ret_lane = v.value_lane_dyn(fx, idx); + ret.write_cvalue(fx, ret_lane); + } + sym::simd_neg | sym::simd_bswap | sym::simd_bitreverse @@ -980,10 +1008,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx.bcx.switch_to_block(if_enabled); let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0); - fx.bcx.ins().jump(next, &[res]); + fx.bcx.ins().jump(next, &[res.into()]); fx.bcx.switch_to_block(if_disabled); - fx.bcx.ins().jump(next, &[val_lane]); + fx.bcx.ins().jump(next, &[val_lane.into()]); fx.bcx.seal_block(next); fx.bcx.switch_to_block(next); @@ -1029,10 +1057,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( ptr_val, Offset32::new(offset), ); - fx.bcx.ins().jump(next, &[res]); + fx.bcx.ins().jump(next, &[res.into()]); fx.bcx.switch_to_block(if_disabled); - fx.bcx.ins().jump(next, &[val_lane]); + fx.bcx.ins().jump(next, &[val_lane.into()]); fx.bcx.seal_block(next); fx.bcx.switch_to_block(next); diff --git a/src/lib.rs b/src/lib.rs index ab09a6f8b38e1..8ef623cde0057 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,9 @@ #![cfg_attr(doc, feature(rustdoc_internals))] // Note: please avoid adding other feature gates where possible #![feature(rustc_private)] +// Only used to define intrinsics in `compiler_builtins.rs`. +#![feature(f16)] +#![feature(f128)] // Note: please avoid adding other feature gates where possible #![warn(rust_2018_idioms)] #![warn(unreachable_pub)] @@ -57,6 +60,7 @@ mod allocator; mod analyze; mod base; mod cast; +mod codegen_f16_f128; mod codegen_i128; mod common; mod compiler_builtins; @@ -76,7 +80,6 @@ mod optimize; mod pointer; mod pretty_clif; mod toolchain; -mod trap; mod unsize; mod unwind_module; mod value_and_place; @@ -198,14 +201,36 @@ impl CodegenBackend for CraneliftCodegenBackend { // FIXME do `unstable_target_features` properly let unstable_target_features = target_features.clone(); + // FIXME(f16_f128): LLVM 20 (currently used by `rustc`) passes `f128` in XMM registers on + // Windows, whereas LLVM 21+ and Cranelift pass it indirectly. This means that `f128` won't + // work when linking against a LLVM-built sysroot. + let has_reliable_f128 = !sess.target.is_like_windows; + let has_reliable_f16 = match &*sess.target.arch { + // FIXME(f16_f128): LLVM 20 does not support `f16` on s390x, meaning the required + // builtins are not available in `compiler-builtins`. + "s390x" => false, + // FIXME(f16_f128): `rustc_codegen_llvm` currently disables support on Windows GNU + // targets due to GCC using a different ABI than LLVM. Therefore `f16` won't be + // available when using a LLVM-built sysroot. + "x86_64" + if sess.target.os == "windows" + && sess.target.env == "gnu" + && sess.target.abi != "llvm" => + { + false + } + _ => true, + }; + TargetConfig { target_features, unstable_target_features, - // Cranelift does not yet support f16 or f128 - has_reliable_f16: false, - has_reliable_f16_math: false, - has_reliable_f128: false, - has_reliable_f128_math: false, + // `rustc_codegen_cranelift` polyfills functionality not yet + // available in Cranelift. + has_reliable_f16, + has_reliable_f16_math: has_reliable_f16, + has_reliable_f128, + has_reliable_f128_math: has_reliable_f128, } } @@ -290,6 +315,12 @@ fn build_isa(sess: &Session, jit: bool) -> Arc { flags_builder.set("enable_llvm_abi_extensions", "true").unwrap(); + if let Some(align) = sess.opts.unstable_opts.min_function_alignment { + flags_builder + .set("log2_min_function_alignment", &align.bytes().ilog2().to_string()) + .unwrap(); + } + use rustc_session::config::OptLevel; match sess.opts.optimize { OptLevel::No => { diff --git a/src/num.rs b/src/num.rs index 2a4d1e3ae571d..f53045df6e799 100644 --- a/src/num.rs +++ b/src/num.rs @@ -1,8 +1,9 @@ //! Various operations on integer and floating-point numbers +use crate::codegen_f16_f128; use crate::prelude::*; -pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> IntCC { +fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> IntCC { use BinOp::*; use IntCC::*; match bin_op { @@ -109,7 +110,7 @@ pub(crate) fn codegen_binop<'tcx>( } } -pub(crate) fn codegen_bool_binop<'tcx>( +fn codegen_bool_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, in_lhs: CValue<'tcx>, @@ -350,25 +351,60 @@ pub(crate) fn codegen_float_binop<'tcx>( let lhs = in_lhs.load_scalar(fx); let rhs = in_rhs.load_scalar(fx); + // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have + // been added to Cranelift. + let (lhs, rhs) = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) { + (codegen_f16_f128::f16_to_f32(fx, lhs), codegen_f16_f128::f16_to_f32(fx, rhs)) + } else { + (lhs, rhs) + }; let b = fx.bcx.ins(); let res = match bin_op { + // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings + // have been added to Cranelift. + BinOp::Add | BinOp::Sub | BinOp::Mul | BinOp::Div + if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F128) => + { + codegen_f16_f128::codegen_f128_binop(fx, bin_op, lhs, rhs) + } BinOp::Add => b.fadd(lhs, rhs), BinOp::Sub => b.fsub(lhs, rhs), BinOp::Mul => b.fmul(lhs, rhs), BinOp::Div => b.fdiv(lhs, rhs), BinOp::Rem => { - let (name, ty) = match in_lhs.layout().ty.kind() { - ty::Float(FloatTy::F32) => ("fmodf", types::F32), - ty::Float(FloatTy::F64) => ("fmod", types::F64), + let (name, ty, lhs, rhs) = match in_lhs.layout().ty.kind() { + ty::Float(FloatTy::F16) => ( + "fmodf", + types::F32, + // FIXME(bytecodealliance/wasmtime#8312): Already converted + // by the FIXME above. + // fx.bcx.ins().fpromote(types::F32, lhs), + // fx.bcx.ins().fpromote(types::F32, rhs), + lhs, + rhs, + ), + ty::Float(FloatTy::F32) => ("fmodf", types::F32, lhs, rhs), + ty::Float(FloatTy::F64) => ("fmod", types::F64, lhs, rhs), + ty::Float(FloatTy::F128) => ("fmodf128", types::F128, lhs, rhs), _ => bug!(), }; - fx.lib_call( + let ret_val = fx.lib_call( name, vec![AbiParam::new(ty), AbiParam::new(ty)], vec![AbiParam::new(ty)], &[lhs, rhs], - )[0] + )[0]; + + let ret_val = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) { + // FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift + // operation once Cranelift backend lowerings have been + // implemented. + codegen_f16_f128::f32_to_f16(fx, ret_val) + } else { + ret_val + }; + return CValue::by_val(ret_val, in_lhs.layout()); } BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => { let fltcc = match bin_op { @@ -380,16 +416,26 @@ pub(crate) fn codegen_float_binop<'tcx>( BinOp::Gt => FloatCC::GreaterThan, _ => unreachable!(), }; - let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs); + // FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift + // `fcmp` once `f16`/`f128` backend lowerings have been added to + // Cranelift. + let val = codegen_f16_f128::fcmp(fx, fltcc, lhs, rhs); return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool)); } _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs), }; + // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have + // been added to Cranelift. + let res = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) { + codegen_f16_f128::f32_to_f16(fx, res) + } else { + res + }; CValue::by_val(res, in_lhs.layout()) } -pub(crate) fn codegen_ptr_binop<'tcx>( +fn codegen_ptr_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, in_lhs: CValue<'tcx>, @@ -457,15 +503,19 @@ pub(crate) fn codegen_ptr_binop<'tcx>( // and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing // a float against itself. Only in case of NaN is it not equal to itself. pub(crate) fn codegen_float_min(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { - let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a); - let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b); + // FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift `fcmp` once + // `f16`/`f128` backend lowerings have been added to Cranelift. + let a_is_nan = codegen_f16_f128::fcmp(fx, FloatCC::NotEqual, a, a); + let a_ge_b = codegen_f16_f128::fcmp(fx, FloatCC::GreaterThanOrEqual, a, b); let temp = fx.bcx.ins().select(a_ge_b, b, a); fx.bcx.ins().select(a_is_nan, b, temp) } pub(crate) fn codegen_float_max(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { - let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a); - let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b); + // FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift `fcmp` once + // `f16`/`f128` backend lowerings have been added to Cranelift. + let a_is_nan = codegen_f16_f128::fcmp(fx, FloatCC::NotEqual, a, a); + let a_le_b = codegen_f16_f128::fcmp(fx, FloatCC::LessThanOrEqual, a, b); let temp = fx.bcx.ins().select(a_le_b, b, a); fx.bcx.ins().select(a_is_nan, b, temp) } diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index cd254b04ed9e1..9400ae9fcff0f 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -8,40 +8,41 @@ //! target x86_64 //! //! function u0:22(i64) -> i8, i8 system_v { -//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd517c453d67c0915E -//! ; instance Instance { def: Item(WithOptConstParam { did: DefId(0:42 ~ example[4e51]::{impl#0}::call_once), const_param_did: None }), args: [ReErased, ReErased] } -//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: Aggregate { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, abi: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), variants: Single { index: 0 } } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd361e9f5c3d1c4deE +//! ; instance Instance { def: Item(DefId(0:42 ~ example[3895]::{impl#0}::call_once)), args: ['{erased}, '{erased}] } +//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } //! //! ; kind loc.idx param pass mode ty -//! ; ssa _0 (u8, u8) 2b 1, 8 var=(0, 1) +//! ; ssa _0 (u8, u8) 2b 1 var=(0, 1) //! ; ret _0 - Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) (u8, u8) //! ; arg _1 - Ignore IsNotEmpty -//! ; arg _2.0 = v0 Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) &&[u16] +//! ; arg _2.0 = v0 Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) &'{erased} &'{erased} [u16] //! -//! ; kind local ty size align (abi,pref) -//! ; zst _1 IsNotEmpty 0b 1, 8 align=8,offset= -//! ; stack _2 (&&[u16],) 8b 8, 8 storage=ss0 -//! ; ssa _3 &mut IsNotEmpty 8b 8, 8 var=2 +//! ; kind local ty size align (abi) +//! ; zst _1 IsNotEmpty 0b 1 align=1,offset= +//! ; stack _2 (&'{erased} &'{erased} [u16],) 8b 8 storage=ss0 +//! ; ssa _3 &'{erased} mut IsNotEmpty 8b 8 var=2 //! -//! ss0 = explicit_slot 16 +//! ss0 = explicit_slot 16, align = 16 //! sig0 = (i64, i64) -> i8, i8 system_v -//! fn0 = colocated u0:23 sig0 ; Instance { def: Item(WithOptConstParam { did: DefId(0:46 ~ example[4e51]::{impl#1}::call_mut), const_param_did: None }), args: [ReErased, ReErased] } +//! fn0 = colocated u0:23 sig0 ; Instance { def: Item(DefId(0:46 ~ example[3895]::{impl#1}::call_mut)), args: ['{erased}, '{erased}] } //! //! block0(v0: i64): //! nop -//! ; write_cvalue: Addr(Pointer { base: Stack(ss0), offset: Offset32(0) }, None): &&[u16] <- ByVal(v0): &&[u16] +//! ; write_cvalue: Addr(Pointer { base: Stack(ss0), offset: Offset32(0) }, None): &'{erased} &'{erased} [u16] <- ByVal(v0): &'{erased} &'{erased} [u16] //! stack_store v0, ss0 //! jump block1 //! //! block1: //! nop //! ; _3 = &mut _1 -//! v1 = iconst.i64 8 -//! ; write_cvalue: Var(_3, var2): &mut IsNotEmpty <- ByVal(v1): &mut IsNotEmpty +//! v1 = iconst.i64 1 +//! ; write_cvalue: Var(_3, var2): &'{erased} mut IsNotEmpty <- ByVal(v1): &'{erased} mut IsNotEmpty //! ; -//! ; _0 = >::call_mut(move _3, _2) +//! ; _0 = >::call_mut(move _3, copy _2) //! v2 = stack_load.i64 ss0 -//! v3, v4 = call fn0(v1, v2) ; v1 = 8 +//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! v3, v4 = call fn0(v1, v2) ; v1 = 1 //! v5 -> v3 //! v6 -> v4 //! ; write_cvalue: VarPair(_0, var0, var1): (u8, u8) <- ByValPair(v3, v4): (u8, u8) @@ -73,6 +74,7 @@ pub(crate) struct CommentWriter { enabled: bool, global_comments: Vec, entity_comments: FxHashMap, + inst_post_comments: FxHashMap, } impl CommentWriter { @@ -95,7 +97,12 @@ impl CommentWriter { vec![] }; - CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() } + CommentWriter { + enabled, + global_comments, + entity_comments: FxHashMap::default(), + inst_post_comments: FxHashMap::default(), + } } } @@ -127,6 +134,25 @@ impl CommentWriter { } } } + + pub(crate) fn add_post_comment + AsRef>( + &mut self, + entity: Inst, + comment: S, + ) { + debug_assert!(self.enabled); + + use std::collections::hash_map::Entry; + match self.inst_post_comments.entry(entity) { + Entry::Occupied(mut occ) => { + occ.get_mut().push('\n'); + occ.get_mut().push_str(comment.as_ref()); + } + Entry::Vacant(vac) => { + vac.insert(comment.into()); + } + } + } } impl FuncWriter for &'_ CommentWriter { @@ -188,10 +214,13 @@ impl FuncWriter for &'_ CommentWriter { inst: Inst, indent: usize, ) -> fmt::Result { - PlainWriter.write_instruction(w, func, aliases, inst, indent)?; if let Some(comment) = self.entity_comments.get(&inst.into()) { writeln!(w, "; {}", comment.replace('\n', "\n; "))?; } + PlainWriter.write_instruction(w, func, aliases, inst, indent)?; + if let Some(comment) = self.inst_post_comments.get(&inst) { + writeln!(w, "; {}", comment.replace('\n', "\n; "))?; + } Ok(()) } } @@ -208,6 +237,14 @@ impl FunctionCx<'_, '_, '_> { ) { self.clif_comments.add_comment(entity, comment); } + + pub(crate) fn add_post_comment + AsRef>( + &mut self, + entity: Inst, + comment: S, + ) { + self.clif_comments.add_post_comment(entity, comment); + } } pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { diff --git a/src/trap.rs b/src/trap.rs deleted file mode 100644 index ac3f58ee1ee6e..0000000000000 --- a/src/trap.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! Helpers used to print a message and abort in case of certain panics and some detected UB. - -use crate::prelude::*; - -fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { - let puts = fx - .module - .declare_function( - "puts", - Linkage::Import, - &Signature { - call_conv: fx.target_config.default_call_conv, - params: vec![AbiParam::new(fx.pointer_type)], - returns: vec![AbiParam::new(types::I32)], - }, - ) - .unwrap(); - let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func); - if fx.clif_comments.enabled() { - fx.add_comment(puts, "puts"); - } - - let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg); - let msg_ptr = fx.anonymous_str(&real_msg); - fx.bcx.ins().call(puts, &[msg_ptr]); -} - -/// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen. -/// -/// Trap code: user65535 -pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef) { - codegen_print(fx, msg.as_ref()); - - let one = fx.bcx.ins().iconst(types::I32, 1); - fx.lib_call("exit", vec![AbiParam::new(types::I32)], vec![], &[one]); - - fx.bcx.ins().trap(TrapCode::user(3).unwrap()); -} diff --git a/src/unwind_module.rs b/src/unwind_module.rs index b950aaa29ce04..b4eb939cf2560 100644 --- a/src/unwind_module.rs +++ b/src/unwind_module.rs @@ -1,10 +1,10 @@ +use cranelift_codegen::Context; use cranelift_codegen::control::ControlPlane; -use cranelift_codegen::ir::{Function, Signature}; +use cranelift_codegen::ir::Signature; use cranelift_codegen::isa::{TargetFrontendConfig, TargetIsa}; -use cranelift_codegen::{Context, FinalizedMachReloc}; use cranelift_module::{ DataDescription, DataId, FuncId, FuncOrDataId, Linkage, Module, ModuleDeclarations, - ModuleResult, + ModuleReloc, ModuleResult, }; use cranelift_object::{ObjectModule, ObjectProduct}; @@ -17,8 +17,8 @@ pub(crate) struct UnwindModule { } impl UnwindModule { - pub(crate) fn new(module: T, pic_eh_frame: bool) -> Self { - let unwind_context = UnwindContext::new(module.isa(), pic_eh_frame); + pub(crate) fn new(mut module: T, pic_eh_frame: bool) -> Self { + let unwind_context = UnwindContext::new(&mut module, pic_eh_frame); UnwindModule { module, unwind_context } } } @@ -33,13 +33,10 @@ impl UnwindModule { #[cfg(feature = "jit")] impl UnwindModule { - pub(crate) fn finalize_definitions(&mut self) { + pub(crate) fn finalize_definitions(mut self) -> cranelift_jit::JITModule { self.module.finalize_definitions().unwrap(); - let prev_unwind_context = std::mem::replace( - &mut self.unwind_context, - UnwindContext::new(self.module.isa(), false), - ); - unsafe { prev_unwind_context.register_jit(&self.module) }; + unsafe { self.unwind_context.register_jit(&self.module) }; + self.module } } @@ -94,17 +91,16 @@ impl Module for UnwindModule { ctrl_plane: &mut ControlPlane, ) -> ModuleResult<()> { self.module.define_function_with_control_plane(func, ctx, ctrl_plane)?; - self.unwind_context.add_function(func, ctx, self.module.isa()); + self.unwind_context.add_function(&mut self.module, func, ctx); Ok(()) } fn define_function_bytes( &mut self, _func_id: FuncId, - _func: &Function, _alignment: u64, _bytes: &[u8], - _relocs: &[FinalizedMachReloc], + _relocs: &[ModuleReloc], ) -> ModuleResult<()> { unimplemented!() } diff --git a/src/value_and_place.rs b/src/value_and_place.rs index f8a19589fdd72..cbfb215a892a4 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -309,6 +309,7 @@ impl<'tcx> CValue<'tcx> { match self.0 { CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(), CValueInner::ByRef(ptr, None) => { + let lane_idx = clif_intcast(fx, lane_idx, fx.pointer_type, false); let field_offset = fx.bcx.ins().imul_imm(lane_idx, lane_layout.size.bytes() as i64); let field_ptr = ptr.offset_value(fx, field_offset); CValue::by_ref(field_ptr, lane_layout) @@ -324,7 +325,7 @@ impl<'tcx> CValue<'tcx> { const_val: ty::ScalarInt, ) -> CValue<'tcx> { assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout); - use cranelift_codegen::ir::immediates::{Ieee32, Ieee64}; + use cranelift_codegen::ir::immediates::{Ieee16, Ieee32, Ieee64, Ieee128}; let clif_ty = fx.clif_type(layout.ty).unwrap(); @@ -345,12 +346,24 @@ impl<'tcx> CValue<'tcx> { let raw_val = const_val.size().truncate(const_val.to_bits(layout.size)); fx.bcx.ins().iconst(clif_ty, raw_val as i64) } + ty::Float(FloatTy::F16) => { + fx.bcx.ins().f16const(Ieee16::with_bits(u16::try_from(const_val).unwrap())) + } ty::Float(FloatTy::F32) => { fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap())) } ty::Float(FloatTy::F64) => { fx.bcx.ins().f64const(Ieee64::with_bits(u64::try_from(const_val).unwrap())) } + ty::Float(FloatTy::F128) => { + let value = fx + .bcx + .func + .dfg + .constants + .insert(Ieee128::with_bits(u128::try_from(const_val).unwrap()).into()); + fx.bcx.ins().f128const(value) + } _ => panic!( "CValue::const_val for non bool/char/float/integer/pointer type {:?} is not allowed", layout.ty @@ -563,27 +576,7 @@ impl<'tcx> CPlace<'tcx> { src_ty, dst_ty, ); - let data = match (src_ty, dst_ty) { - (_, _) if src_ty == dst_ty => data, - - // This is a `write_cvalue_transmute`. - (types::I32, types::F32) - | (types::F32, types::I32) - | (types::I64, types::F64) - | (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data), - _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data), - _ if src_ty.is_vector() || dst_ty.is_vector() => { - // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers. - let ptr = fx.create_stack_slot(src_ty.bytes(), src_ty.bytes()); - ptr.store(fx, data, MemFlags::trusted()); - ptr.load(fx, dst_ty, MemFlags::trusted()) - } - - // `CValue`s should never contain SSA-only types, so if you ended - // up here having seen an error like `B1 -> I8`, then before - // calling `write_cvalue` you need to add a `bint` instruction. - _ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty), - }; + let data = if src_ty == dst_ty { data } else { codegen_bitcast(fx, dst_ty, data) }; //fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index())); fx.bcx.def_var(var, data); } @@ -591,13 +584,9 @@ impl<'tcx> CPlace<'tcx> { assert_eq!(self.layout().size, from.layout().size); if fx.clif_comments.enabled() { - use cranelift_codegen::cursor::{Cursor, CursorPosition}; - let cur_block = match fx.bcx.cursor().position() { - CursorPosition::After(block) => block, - _ => unreachable!(), - }; - fx.add_comment( - fx.bcx.func.layout.last_inst(cur_block).unwrap(), + let inst = fx.bcx.func.layout.last_inst(fx.bcx.current_block().unwrap()).unwrap(); + fx.add_post_comment( + inst, format!( "{}: {:?}: {:?} <- {:?}: {:?}", method, @@ -806,6 +795,35 @@ impl<'tcx> CPlace<'tcx> { } } + /// Write a value to an individual lane in a SIMD vector. + pub(crate) fn write_lane_dyn( + self, + fx: &mut FunctionCx<'_, '_, 'tcx>, + lane_idx: Value, + value: CValue<'tcx>, + ) { + let layout = self.layout(); + assert!(layout.ty.is_simd()); + let (_lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let lane_layout = fx.layout_of(lane_ty); + assert_eq!(lane_layout, value.layout()); + + match self.inner { + CPlaceInner::Var(_, _) => unreachable!(), + CPlaceInner::VarPair(_, _, _) => unreachable!(), + CPlaceInner::Addr(ptr, None) => { + let lane_idx = clif_intcast(fx, lane_idx, fx.pointer_type, false); + let field_offset = fx + .bcx + .ins() + .imul_imm(lane_idx, i64::try_from(lane_layout.size.bytes()).unwrap()); + let field_ptr = ptr.offset_value(fx, field_offset); + CPlace::for_ptr(field_ptr, lane_layout).write_cvalue(fx, value); + } + CPlaceInner::Addr(_, Some(_)) => unreachable!(), + } + } + pub(crate) fn place_index( self, fx: &mut FunctionCx<'_, '_, 'tcx>, diff --git a/triagebot.toml b/triagebot.toml new file mode 100644 index 0000000000000..13da0a87def3c --- /dev/null +++ b/triagebot.toml @@ -0,0 +1,7 @@ +# Documentation at https://forge.rust-lang.org/triagebot/index.html + +# Prevents un-canonicalized issue links (to avoid wrong issues being linked in r-l/rust) +[issue-links] + +# Prevents mentions in commits to avoid users being spammed +[no-mentions] From ea26272c94c7a66ed018930c13c63551cb2f1390 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sun, 18 May 2025 17:12:13 +0000 Subject: [PATCH 02/38] clif: Provide better output messages for failed copies --- build_system/utils.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build_system/utils.rs b/build_system/utils.rs index f239976845964..d9807155a3d5d 100644 --- a/build_system/utils.rs +++ b/build_system/utils.rs @@ -213,11 +213,13 @@ pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) { if filename == "." || filename == ".." { continue; } + let src = from.join(&filename); + let dst = to.join(&filename); if entry.metadata().unwrap().is_dir() { - fs::create_dir(to.join(&filename)).unwrap(); - copy_dir_recursively(&from.join(&filename), &to.join(&filename)); + fs::create_dir(&dst).unwrap_or_else(|e| panic!("failed to create {dst:?}: {e}")); + copy_dir_recursively(&src, &dst); } else { - fs::copy(from.join(&filename), to.join(&filename)).unwrap(); + fs::copy(&src, &dst).unwrap_or_else(|e| panic!("failed to copy {src:?}->{dst:?}: {e}")); } } } From 99e783d4e9bdcc44e2071973631a7d5e689b0787 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 24 May 2025 16:35:20 +0200 Subject: [PATCH 03/38] atomic_load intrinsic: use const generic parameter for ordering --- src/intrinsics/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index b21ca32c9a2e3..0de23e55e8174 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -870,11 +870,12 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME use a compiler fence once Cranelift supports it fx.bcx.ins().fence(); } - _ if intrinsic.as_str().starts_with("atomic_load") => { + sym::atomic_load => { intrinsic_args!(fx, args => (ptr); intrinsic); let ptr = ptr.load_scalar(fx); let ty = generic_args.type_at(0); + let _ord = generic_args.const_at(1).to_value(); // FIXME: forward this to cranelift once they support that match ty.kind() { ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { // FIXME implement 128bit atomics From bc74556fa4dd083eeb0fd7177edf5e64b89763a0 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 31 May 2025 13:43:30 +0000 Subject: [PATCH 04/38] Rustup to rustc 1.89.0-nightly (70b3f4666 2025-05-30) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index af4bd6dc6b851..bc7da7c12a050 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-05-25" +channel = "nightly-2025-05-31" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From 814ef9f080907942dd3884a536c0a71212c8c3be Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:51:18 +0000 Subject: [PATCH 05/38] Rustup to rustc 1.89.0-nightly (5d707b07e 2025-06-02) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index bc7da7c12a050..f20b6b82a53a7 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-05-31" +channel = "nightly-2025-06-03" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From ad43500e96c4dd6a90e66cc7fdc018f1a0317a0c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 30 May 2025 11:55:06 +0000 Subject: [PATCH 06/38] Only borrow EncodedMetadata in codegen_crate And move passing it to the linker to the driver code. --- src/driver/aot.rs | 13 ++----------- src/lib.rs | 9 ++------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 5d07c94859f33..58a0a38094520 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -62,7 +62,6 @@ pub(crate) struct OngoingCodegen { modules: Vec, allocator_module: Option, metadata_module: Option, - metadata: EncodedMetadata, crate_info: CrateInfo, concurrency_limiter: ConcurrencyLimiter, } @@ -135,7 +134,6 @@ impl OngoingCodegen { modules, allocator_module: self.allocator_module, metadata_module: self.metadata_module, - metadata: self.metadata, crate_info: self.crate_info, }; @@ -706,11 +704,7 @@ fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option { } } -pub(crate) fn run_aot( - tcx: TyCtxt<'_>, - metadata: EncodedMetadata, - need_metadata_module: bool, -) -> Box { +pub(crate) fn run_aot(tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Box { // FIXME handle `-Ctarget-cpu=native` let target_cpu = match tcx.sess.opts.cg.target_cpu { Some(ref name) => name, @@ -727,7 +721,6 @@ pub(crate) fn run_aot( modules: vec![], allocator_module: None, metadata_module: None, - metadata, crate_info: CrateInfo::new(tcx, target_cpu), concurrency_limiter: ConcurrencyLimiter::new(0), }); @@ -787,14 +780,12 @@ pub(crate) fn run_aot( let allocator_module = emit_allocator_module(tcx); - let metadata_module = - if need_metadata_module { Some(emit_metadata_module(tcx, &metadata)) } else { None }; + let metadata_module = metadata.map(|metadata| emit_metadata_module(tcx, metadata)); Box::new(OngoingCodegen { modules, allocator_module, metadata_module, - metadata, crate_info: CrateInfo::new(tcx, target_cpu), concurrency_limiter: concurrency_limiter.0, }) diff --git a/src/lib.rs b/src/lib.rs index 8ef623cde0057..277d4f16f1945 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -238,12 +238,7 @@ impl CodegenBackend for CraneliftCodegenBackend { println!("Cranelift version: {}", cranelift_codegen::VERSION); } - fn codegen_crate( - &self, - tcx: TyCtxt<'_>, - metadata: EncodedMetadata, - need_metadata_module: bool, - ) -> Box { + fn codegen_crate(&self, tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Box { info!("codegen crate {}", tcx.crate_name(LOCAL_CRATE)); let config = self.config.clone().unwrap_or_else(|| { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) @@ -256,7 +251,7 @@ impl CodegenBackend for CraneliftCodegenBackend { #[cfg(not(feature = "jit"))] tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift"); } else { - driver::aot::run_aot(tcx, metadata, need_metadata_module) + driver::aot::run_aot(tcx, metadata) } } From 2974c99566b343afc4834db7af525c4f82224b68 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 30 May 2025 12:51:15 +0000 Subject: [PATCH 07/38] Move metadata object generation for dylibs to the linker code This deduplicates some code between codegen backends and may in the future allow adding extra metadata that is only known at link time. --- src/driver/aot.rs | 46 +--------------------------------------------- src/lib.rs | 5 ++--- 2 files changed, 3 insertions(+), 48 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 58a0a38094520..442151fe32de8 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -11,7 +11,6 @@ use std::thread::JoinHandle; use cranelift_object::{ObjectBuilder, ObjectModule}; use rustc_codegen_ssa::assert_module_sources::CguReuse; use rustc_codegen_ssa::back::link::ensure_removed; -use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file; use rustc_codegen_ssa::base::determine_cgu_reuse; use rustc_codegen_ssa::{ CodegenResults, CompiledModule, CrateInfo, ModuleKind, errors as ssa_errors, @@ -19,7 +18,6 @@ use rustc_codegen_ssa::{ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; -use rustc_metadata::EncodedMetadata; use rustc_metadata::fs::copy_to_stdout; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; @@ -61,7 +59,6 @@ impl HashStable for OngoingModuleCodegen { pub(crate) struct OngoingCodegen { modules: Vec, allocator_module: Option, - metadata_module: Option, crate_info: CrateInfo, concurrency_limiter: ConcurrencyLimiter, } @@ -133,7 +130,6 @@ impl OngoingCodegen { let codegen_results = CodegenResults { modules, allocator_module: self.allocator_module, - metadata_module: self.metadata_module, crate_info: self.crate_info, }; @@ -644,42 +640,6 @@ fn module_codegen( })) } -fn emit_metadata_module(tcx: TyCtxt<'_>, metadata: &EncodedMetadata) -> CompiledModule { - use rustc_middle::mir::mono::CodegenUnitNameBuilder; - - let _timer = tcx.sess.timer("write compressed metadata"); - - let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx); - let metadata_cgu_name = cgu_name_builder - .build_cgu_name(LOCAL_CRATE, ["crate"], Some("metadata")) - .as_str() - .to_string(); - - let tmp_file = tcx.output_filenames(()).temp_path_for_cgu( - OutputType::Metadata, - &metadata_cgu_name, - tcx.sess.invocation_temp.as_deref(), - ); - - let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx); - let obj = create_compressed_metadata_file(tcx.sess, metadata, &symbol_name); - - if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.dcx().fatal(format!("error writing metadata object file: {}", err)); - } - - CompiledModule { - name: metadata_cgu_name, - kind: ModuleKind::Metadata, - object: Some(tmp_file), - dwarf_object: None, - bytecode: None, - assembly: None, - llvm_ir: None, - links_from_incr_cache: Vec::new(), - } -} - fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option { let mut allocator_module = make_module(tcx.sess, "allocator_shim".to_string()); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module); @@ -704,7 +664,7 @@ fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option { } } -pub(crate) fn run_aot(tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Box { +pub(crate) fn run_aot(tcx: TyCtxt<'_>) -> Box { // FIXME handle `-Ctarget-cpu=native` let target_cpu = match tcx.sess.opts.cg.target_cpu { Some(ref name) => name, @@ -720,7 +680,6 @@ pub(crate) fn run_aot(tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Bo return Box::new(OngoingCodegen { modules: vec![], allocator_module: None, - metadata_module: None, crate_info: CrateInfo::new(tcx, target_cpu), concurrency_limiter: ConcurrencyLimiter::new(0), }); @@ -780,12 +739,9 @@ pub(crate) fn run_aot(tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Bo let allocator_module = emit_allocator_module(tcx); - let metadata_module = metadata.map(|metadata| emit_metadata_module(tcx, metadata)); - Box::new(OngoingCodegen { modules, allocator_module, - metadata_module, crate_info: CrateInfo::new(tcx, target_cpu), concurrency_limiter: concurrency_limiter.0, }) diff --git a/src/lib.rs b/src/lib.rs index 277d4f16f1945..07ea29f3024ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,7 +46,6 @@ use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::{CodegenResults, TargetConfig}; -use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_session::Session; use rustc_session::config::OutputFilenames; @@ -238,7 +237,7 @@ impl CodegenBackend for CraneliftCodegenBackend { println!("Cranelift version: {}", cranelift_codegen::VERSION); } - fn codegen_crate(&self, tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Box { + fn codegen_crate(&self, tcx: TyCtxt<'_>) -> Box { info!("codegen crate {}", tcx.crate_name(LOCAL_CRATE)); let config = self.config.clone().unwrap_or_else(|| { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) @@ -251,7 +250,7 @@ impl CodegenBackend for CraneliftCodegenBackend { #[cfg(not(feature = "jit"))] tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift"); } else { - driver::aot::run_aot(tcx, metadata) + driver::aot::run_aot(tcx) } } From b97136c817c989f37baca1191a07828346251a64 Mon Sep 17 00:00:00 2001 From: sayantn Date: Tue, 3 Jun 2025 20:37:10 +0530 Subject: [PATCH 08/38] Add impl for `llvm.roundeven` in cg_clif - remove unused `llvm.aarch64.neon.frintn` from cg_clif --- example/neon.rs | 2 +- src/intrinsics/llvm.rs | 8 ++++++++ src/intrinsics/llvm_aarch64.rs | 8 -------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/example/neon.rs b/example/neon.rs index 69ce17d3d752a..704f866e2c4f5 100644 --- a/example/neon.rs +++ b/example/neon.rs @@ -233,7 +233,7 @@ unsafe fn test_vaddvq_f32() { #[cfg(target_arch = "aarch64")] unsafe fn test_vrndnq_f32() { - // AArch64 llvm intrinsic: llvm.aarch64.neon.frintn.v4f32 + // llvm intrinsic: llvm.roundeven.v4f32 let a = f32x4::from([0.1, -1.9, 4.5, 5.5]); let e = f32x4::from([0., -2., 4., 6.]); let r: f32x4 = transmute(vrndnq_f32(transmute(a))); diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 2e02e85a997d5..81dc038a55166 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -62,6 +62,14 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( }); } + _ if intrinsic.starts_with("llvm.roundeven.v") => { + intrinsic_args!(fx, args => (v); intrinsic); + + simd_for_each_lane(fx, v, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().nearest(lane) + }); + } + _ => { fx.tcx .dcx() diff --git a/src/intrinsics/llvm_aarch64.rs b/src/intrinsics/llvm_aarch64.rs index d22483cf17764..5448f1b1e9097 100644 --- a/src/intrinsics/llvm_aarch64.rs +++ b/src/intrinsics/llvm_aarch64.rs @@ -264,14 +264,6 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().fadd(a, b)); } - _ if intrinsic.starts_with("llvm.aarch64.neon.frintn.v") => { - intrinsic_args!(fx, args => (v); intrinsic); - - simd_for_each_lane(fx, v, ret, &|fx, _lane_ty, _res_lane_ty, lane| { - fx.bcx.ins().nearest(lane) - }); - } - _ if intrinsic.starts_with("llvm.aarch64.neon.smaxv.i") => { intrinsic_args!(fx, args => (v); intrinsic); From 12be26f9280b2a72918be0e7a642a1267ea28896 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 21 May 2025 23:29:37 +0200 Subject: [PATCH 09/38] cg_clif: convert to CanonAbi --- src/abi/mod.rs | 49 ++++++++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 5f7bf3821d77d..fe5b220117f3c 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -10,7 +10,7 @@ use std::mem; use cranelift_codegen::ir::{ArgumentPurpose, SigRef}; use cranelift_codegen::isa::CallConv; use cranelift_module::ModuleError; -use rustc_abi::ExternAbi; +use rustc_abi::{CanonAbi, ExternAbi, X86Call}; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; @@ -19,7 +19,7 @@ use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_session::Session; use rustc_span::source_map::Spanned; -use rustc_target::callconv::{Conv, FnAbi, PassMode}; +use rustc_target::callconv::{FnAbi, PassMode}; use smallvec::SmallVec; use self::pass_mode::*; @@ -42,32 +42,27 @@ fn clif_sig_from_fn_abi<'tcx>( Signature { params, returns, call_conv } } -pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv { +pub(crate) fn conv_to_call_conv( + sess: &Session, + c: CanonAbi, + default_call_conv: CallConv, +) -> CallConv { match c { - Conv::Rust | Conv::C => default_call_conv, - Conv::Cold | Conv::PreserveMost | Conv::PreserveAll => CallConv::Cold, - Conv::X86_64SysV => CallConv::SystemV, - Conv::X86_64Win64 => CallConv::WindowsFastcall, - - // Should already get a back compat warning - Conv::X86Fastcall | Conv::X86Stdcall | Conv::X86ThisCall | Conv::X86VectorCall => { - default_call_conv - } - - Conv::X86Intr | Conv::RiscvInterrupt { .. } => { - sess.dcx().fatal(format!("interrupt call conv {c:?} not yet implemented")) + CanonAbi::Rust | CanonAbi::C => default_call_conv, + CanonAbi::RustCold => CallConv::Cold, + + CanonAbi::X86(x86_call) => match x86_call { + X86Call::SysV64 => CallConv::SystemV, + X86Call::Win64 => CallConv::WindowsFastcall, + // Should already get a back compat warning + _ => default_call_conv, + }, + + CanonAbi::Interrupt(_) | CanonAbi::Arm(_) => { + sess.dcx().fatal("call conv {c:?} is not yet implemented") } - - Conv::ArmAapcs => sess.dcx().fatal("aapcs call conv not yet implemented"), - Conv::CCmseNonSecureCall => { - sess.dcx().fatal("C-cmse-nonsecure-call call conv is not yet implemented"); - } - Conv::CCmseNonSecureEntry => { - sess.dcx().fatal("C-cmse-nonsecure-entry call conv is not yet implemented"); - } - - Conv::Msp430Intr | Conv::GpuKernel | Conv::AvrInterrupt | Conv::AvrNonBlockingInterrupt => { - unreachable!("tried to use {c:?} call conv which only exists on an unsupported target"); + CanonAbi::GpuKernel => { + unreachable!("tried to use {c:?} call conv which only exists on an unsupported target") } } } @@ -610,7 +605,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( target: CallTarget, call_args: &mut Vec, ) { - if fn_abi.conv != Conv::C { + if fn_abi.conv != CanonAbi::C { fx.tcx.dcx().span_fatal( source_info.span, format!("Variadic call for non-C abi {:?}", fn_abi.conv), From 57d0559160081a1db2163b06e3fe622bf129f5c5 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Tue, 3 Jun 2025 23:42:21 -0700 Subject: [PATCH 10/38] Change `tag_field` to `FieldIdx` in `Variants::Multiple` It was already available as a generic parameter anyway, and it's not like we'll ever put a tag in the 5-billionth field. --- src/discriminant.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/discriminant.rs b/src/discriminant.rs index 4d0d5dc60eba6..a08b0e0cbfc59 100644 --- a/src/discriminant.rs +++ b/src/discriminant.rs @@ -28,7 +28,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>( tag_encoding: TagEncoding::Direct, variants: _, } => { - let ptr = place.place_field(fx, FieldIdx::new(tag_field)); + let ptr = place.place_field(fx, tag_field); let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val; let to = match ptr.layout().ty.kind() { ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { @@ -53,7 +53,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>( variants: _, } => { if variant_index != untagged_variant { - let niche = place.place_field(fx, FieldIdx::new(tag_field)); + let niche = place.place_field(fx, tag_field); let niche_type = fx.clif_type(niche.layout().ty).unwrap(); let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); let niche_value = (niche_value as u128).wrapping_add(niche_start); @@ -118,7 +118,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>( let cast_to = fx.clif_type(dest_layout.ty).unwrap(); // Read the tag/niche-encoded discriminant from memory. - let tag = value.value_field(fx, FieldIdx::new(tag_field)); + let tag = value.value_field(fx, tag_field); let tag = tag.load_scalar(fx); // Decode the discriminant (specifically if it's niche-encoded). From bcac194b240879545379def996dd9123dfc829ee Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 5 Jun 2025 09:35:09 +0000 Subject: [PATCH 11/38] Rustup to rustc 1.89.0-nightly (4b27a04cc 2025-06-04) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index f20b6b82a53a7..a9a6c7bdf4345 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-03" +channel = "nightly-2025-06-05" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From 59549af369892581543afc0b48c73a24dbf87e7a Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Wed, 4 Jun 2025 06:26:56 +0000 Subject: [PATCH 12/38] Replace some `Option` with `Span` and use DUMMY_SP instead of None --- src/base.rs | 16 ++++++++-------- src/intrinsics/llvm.rs | 2 +- src/intrinsics/llvm_aarch64.rs | 2 +- src/intrinsics/llvm_x86.rs | 2 +- src/intrinsics/mod.rs | 6 +++--- src/main_shim.rs | 4 ++-- src/num.rs | 2 +- src/unsize.rs | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/base.rs b/src/base.rs index 4617304105a5b..0b641ba64b7a4 100644 --- a/src/base.rs +++ b/src/base.rs @@ -380,7 +380,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { rustc_hir::LangItem::PanicBoundsCheck, &[index, len, location], *unwind, - Some(source_info.span), + source_info.span, ); } AssertKind::MisalignedPointerDereference { ref required, ref found } => { @@ -393,7 +393,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { rustc_hir::LangItem::PanicMisalignedPointerDereference, &[required, found, location], *unwind, - Some(source_info.span), + source_info.span, ); } AssertKind::NullPointerDereference => { @@ -404,7 +404,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { rustc_hir::LangItem::PanicNullPointerDereference, &[location], *unwind, - Some(source_info.span), + source_info.span, ) } _ => { @@ -415,7 +415,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { msg.panic_function(), &[location], *unwind, - Some(source_info.span), + source_info.span, ); } } @@ -531,7 +531,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { ); } TerminatorKind::UnwindTerminate(reason) => { - codegen_unwind_terminate(fx, Some(source_info.span), *reason); + codegen_unwind_terminate(fx, source_info.span, *reason); } TerminatorKind::UnwindResume => { // FIXME implement unwinding @@ -1074,7 +1074,7 @@ pub(crate) fn codegen_operand<'tcx>( pub(crate) fn codegen_panic_nounwind<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, - span: Option, + span: Span, ) { let msg_ptr = fx.anonymous_str(msg_str); let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); @@ -1091,7 +1091,7 @@ pub(crate) fn codegen_panic_nounwind<'tcx>( pub(crate) fn codegen_unwind_terminate<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, - span: Option, + span: Span, reason: UnwindTerminateReason, ) { codegen_panic_inner(fx, reason.lang_item(), &[], UnwindAction::Unreachable, span); @@ -1102,7 +1102,7 @@ fn codegen_panic_inner<'tcx>( lang_item: rustc_hir::LangItem, args: &[Value], _unwind: UnwindAction, - span: Option, + span: Span, ) { fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 2e02e85a997d5..99a5518d0b688 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -71,7 +71,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\ Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues" ); - crate::base::codegen_panic_nounwind(fx, &msg, None); + crate::base::codegen_panic_nounwind(fx, &msg, span); return; } } diff --git a/src/intrinsics/llvm_aarch64.rs b/src/intrinsics/llvm_aarch64.rs index d22483cf17764..c22f2a7b8734a 100644 --- a/src/intrinsics/llvm_aarch64.rs +++ b/src/intrinsics/llvm_aarch64.rs @@ -512,7 +512,7 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\ Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues" ); - crate::base::codegen_panic_nounwind(fx, &msg, None); + crate::base::codegen_panic_nounwind(fx, &msg, fx.mir.span); return; } } diff --git a/src/intrinsics/llvm_x86.rs b/src/intrinsics/llvm_x86.rs index 3d67913a8fff1..615f6c47d902f 100644 --- a/src/intrinsics/llvm_x86.rs +++ b/src/intrinsics/llvm_x86.rs @@ -1321,7 +1321,7 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\ Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues" ); - crate::base::codegen_panic_nounwind(fx, &msg, None); + crate::base::codegen_panic_nounwind(fx, &msg, span); return; } } diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 0de23e55e8174..27a5df8b1520a 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -785,7 +785,7 @@ fn codegen_regular_intrinsic_call<'tcx>( } }) }); - crate::base::codegen_panic_nounwind(fx, &msg_str, Some(source_info.span)); + crate::base::codegen_panic_nounwind(fx, &msg_str, source_info.span); return Ok(()); } } @@ -884,7 +884,7 @@ fn codegen_regular_intrinsic_call<'tcx>( crate::base::codegen_panic_nounwind( fx, "128bit atomics not yet supported", - None, + source_info.span, ); return Ok(()); } else { @@ -919,7 +919,7 @@ fn codegen_regular_intrinsic_call<'tcx>( crate::base::codegen_panic_nounwind( fx, "128bit atomics not yet supported", - None, + source_info.span, ); return Ok(()); } else { diff --git a/src/main_shim.rs b/src/main_shim.rs index 6eef97c14dd28..bf756860b6499 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -101,7 +101,7 @@ pub(crate) fn maybe_create_entry_wrapper( let call_inst = bcx.ins().call(main_func_ref, &[]); let call_results = bcx.func.dfg.inst_results(call_inst).to_owned(); - let termination_trait = tcx.require_lang_item(LangItem::Termination, None); + let termination_trait = tcx.require_lang_item(LangItem::Termination, DUMMY_SP); let report = tcx .associated_items(termination_trait) .find_by_ident_and_kind( @@ -136,7 +136,7 @@ pub(crate) fn maybe_create_entry_wrapper( } } else { // Regular main fn invoked via start lang item. - let start_def_id = tcx.require_lang_item(LangItem::Start, None); + let start_def_id = tcx.require_lang_item(LangItem::Start, DUMMY_SP); let start_instance = Instance::expect_resolve( tcx, ty::TypingEnv::fully_monomorphized(), diff --git a/src/num.rs b/src/num.rs index f53045df6e799..95d44dfb6d950 100644 --- a/src/num.rs +++ b/src/num.rs @@ -54,7 +54,7 @@ fn codegen_three_way_compare<'tcx>( let gt = fx.bcx.ins().icmp(gt_cc, lhs, rhs); let lt = fx.bcx.ins().icmp(lt_cc, lhs, rhs); let val = fx.bcx.ins().isub(gt, lt); - CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(Some(fx.mir.span)))) + CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(fx.mir.span))) } fn codegen_compare_bin_op<'tcx>( diff --git a/src/unsize.rs b/src/unsize.rs index f8bbb21492015..662546e499988 100644 --- a/src/unsize.rs +++ b/src/unsize.rs @@ -240,7 +240,7 @@ pub(crate) fn size_and_align_of<'tcx>( }) }); - codegen_panic_nounwind(fx, &msg_str, None); + codegen_panic_nounwind(fx, &msg_str, fx.mir.span); fx.bcx.switch_to_block(next_block); From 2192d869d0624a2013e45183db1cf3595aa7d989 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Wed, 4 Jun 2025 01:44:40 -0700 Subject: [PATCH 13/38] Update `InterpCx::project_field` to take `FieldIdx` As suggested by Ralf in 142005. --- src/inline_asm.rs | 2 +- src/vtable.rs | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/inline_asm.rs b/src/inline_asm.rs index afee50955497c..120d6ff9e38e0 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -850,7 +850,7 @@ fn asm_clif_type<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> Option { let fields = &adt.non_enum_variant().fields; - let ty = fields[FieldIdx::from_u32(1)].ty(fx.tcx, args); + let ty = fields[FieldIdx::ONE].ty(fx.tcx, args); let ty::Adt(ty, args) = ty.kind() else { unreachable!("expected first field of `MaybeUninit` to be an ADT") }; diff --git a/src/vtable.rs b/src/vtable.rs index 9d9e0462a9b72..05a8e3c334216 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -53,7 +53,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>( .layout() .non_1zst_field(fx) .expect("not exactly one non-1-ZST field in a `DispatchFromDyn` type"); - arg = arg.value_field(fx, FieldIdx::new(idx)); + arg = arg.value_field(fx, idx); } } @@ -62,8 +62,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>( let inner_layout = fx.layout_of(arg.layout().ty.builtin_deref(true).unwrap()); let dyn_star = CPlace::for_ptr(Pointer::new(arg.load_scalar(fx)), inner_layout); let ptr = dyn_star.place_field(fx, FieldIdx::ZERO).to_ptr(); - let vtable = - dyn_star.place_field(fx, FieldIdx::new(1)).to_cvalue(fx).load_scalar(fx); + let vtable = dyn_star.place_field(fx, FieldIdx::ONE).to_cvalue(fx).load_scalar(fx); break 'block (ptr, vtable); } } From 4e3bf2da2f2ec55f16b5e141bc01ce10f38ee054 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 7 Jun 2025 14:18:29 +0000 Subject: [PATCH 14/38] Rustup to rustc 1.89.0-nightly (44f415c1d 2025-06-06) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index a9a6c7bdf4345..9bc06fcc838a1 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-05" +channel = "nightly-2025-06-07" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From 18b12218a4ae486b3c699efa159ab07277624982 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 28 May 2025 18:37:58 +0200 Subject: [PATCH 15/38] intrinsics: use const generic to set atomic ordering --- src/intrinsics/mod.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 27a5df8b1520a..a0f96d85dc3c1 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -875,7 +875,6 @@ fn codegen_regular_intrinsic_call<'tcx>( let ptr = ptr.load_scalar(fx); let ty = generic_args.type_at(0); - let _ord = generic_args.const_at(1).to_value(); // FIXME: forward this to cranelift once they support that match ty.kind() { ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { // FIXME implement 128bit atomics @@ -906,7 +905,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(ty)); ret.write_cvalue(fx, val); } - _ if intrinsic.as_str().starts_with("atomic_store") => { + sym::atomic_store => { intrinsic_args!(fx, args => (ptr, val); intrinsic); let ptr = ptr.load_scalar(fx); @@ -939,7 +938,7 @@ fn codegen_regular_intrinsic_call<'tcx>( fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); } - _ if intrinsic.as_str().starts_with("atomic_xchg") => { + sym::atomic_xchg => { intrinsic_args!(fx, args => (ptr, new); intrinsic); let ptr = ptr.load_scalar(fx); @@ -960,8 +959,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_cxchg") => { - // both atomic_cxchg_* and atomic_cxchgweak_* + sym::atomic_cxchg | sym::atomic_cxchgweak => { intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic); let ptr = ptr.load_scalar(fx); @@ -984,7 +982,7 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, ret_val) } - _ if intrinsic.as_str().starts_with("atomic_xadd") => { + sym::atomic_xadd => { intrinsic_args!(fx, args => (ptr, amount); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1006,7 +1004,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_xsub") => { + sym::atomic_xsub => { intrinsic_args!(fx, args => (ptr, amount); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1028,7 +1026,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_and") => { + sym::atomic_and => { intrinsic_args!(fx, args => (ptr, src); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1049,7 +1047,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_or") => { + sym::atomic_or => { intrinsic_args!(fx, args => (ptr, src); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1070,7 +1068,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_xor") => { + sym::atomic_xor => { intrinsic_args!(fx, args => (ptr, src); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1091,7 +1089,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_nand") => { + sym::atomic_nand => { intrinsic_args!(fx, args => (ptr, src); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1112,7 +1110,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_max") => { + sym::atomic_max => { intrinsic_args!(fx, args => (ptr, src); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1133,7 +1131,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_umax") => { + sym::atomic_umax => { intrinsic_args!(fx, args => (ptr, src); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1154,7 +1152,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_min") => { + sym::atomic_min => { intrinsic_args!(fx, args => (ptr, src); intrinsic); let ptr = ptr.load_scalar(fx); @@ -1175,7 +1173,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); } - _ if intrinsic.as_str().starts_with("atomic_umin") => { + sym::atomic_umin => { intrinsic_args!(fx, args => (ptr, src); intrinsic); let ptr = ptr.load_scalar(fx); From 7988b8c0b3cec24f1c827c885f418fa9ae3aa6a6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 8 Jun 2025 11:42:13 +0000 Subject: [PATCH 16/38] Rustup to rustc 1.89.0-nightly (cdd545be1 2025-06-07) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 9bc06fcc838a1..1e867bd76580a 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-07" +channel = "nightly-2025-06-08" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From d5e9833af388c9f3122877fe355ff7b244c50516 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Tue, 11 Feb 2025 22:09:16 -0800 Subject: [PATCH 17/38] Remove rustc's notion of "preferred" alignment AKA `__alignof` In PR 90877 T-lang decided not to remove `intrinsics::pref_align_of`. However, the intrinsic and its supporting code 1. is a nightly feature, so can be removed at compiler/libs discretion 2. requires considerable effort in the compiler to support, as it necessarily complicates every single site reasoning about alignment 3. has been justified based on relevance to codegen, but it is only a requirement for C++ (not C, not Rust) stack frame layout for AIX, in ways Rust would not consider even with increased C++ interop 4. is only used by rustc to overalign some globals, not correctness 5. can be adequately replaced by other rules for globals, as it mostly affects alignments for a few types under 16 bytes of alignment 6. has only one clear benefactor: automating C -> Rust translation for GNU extensions like `__alignof` 7. such code was likely intended to be `alignof` or `_Alignof`, because the GNU extension is a "false friend" of the C keyword, which makes the choice to support such a mapping very questionable 8. makes it easy to do incorrect codegen in the compiler by its mere presence as usual Rust rules of alignment (e.g. `size == align * N`) do not hold with preferred alignment The implementation is clearly damaging the code quality of the compiler. Thus it is within the compiler team's purview to simply rip it out. If T-lang wishes to have this intrinsic restored for c2rust's benefit, it would have to use a radically different implementation that somehow does not cause internal incorrectness. Until then, remove the intrinsic and its supporting code, as one tool and an ill-considered GCC extension cannot justify risking correctness. Because we touch a fair amount of the compiler to change this at all, and unfortunately the duplication of AbiAndPrefAlign is deep-rooted, we keep an "AbiAlign" type which we can wean code off later. --- src/intrinsics/mod.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index a0f96d85dc3c1..1d1cf884e48b2 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -812,11 +812,7 @@ fn codegen_regular_intrinsic_call<'tcx>( dest.write_cvalue(fx, val); } - sym::pref_align_of - | sym::needs_drop - | sym::type_id - | sym::type_name - | sym::variant_count => { + sym::needs_drop | sym::type_id | sym::type_name | sym::variant_count => { intrinsic_args!(fx, args => (); intrinsic); let const_val = fx From 30c48bcb55e10518e4e59a78dcab439a0b327725 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 10 Jun 2025 09:39:50 +0000 Subject: [PATCH 18/38] Use Operand::constant() in a couple of places Also reduce visibility of a function --- src/constant.rs | 2 +- src/intrinsics/llvm_x86.rs | 7 ++++--- src/intrinsics/simd.rs | 7 ++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/constant.rs b/src/constant.rs index c8527c3a57dfe..3a62cd52a9d94 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -228,7 +228,7 @@ fn pointer_for_allocation<'tcx>( crate::pointer::Pointer::new(global_ptr) } -pub(crate) fn data_id_for_alloc_id( +fn data_id_for_alloc_id( cx: &mut ConstantCx, module: &mut dyn Module, alloc_id: AllocId, diff --git a/src/intrinsics/llvm_x86.rs b/src/intrinsics/llvm_x86.rs index 615f6c47d902f..37fbe4be1b0fe 100644 --- a/src/intrinsics/llvm_x86.rs +++ b/src/intrinsics/llvm_x86.rs @@ -202,9 +202,10 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( }; let x = codegen_operand(fx, &x.node); let y = codegen_operand(fx, &y.node); - let kind = match &kind.node { - Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0, - Operand::Copy(_) | Operand::Move(_) => unreachable!("{kind:?}"), + let kind = if let Some(const_) = kind.node.constant() { + crate::constant::eval_mir_constant(fx, const_).0 + } else { + unreachable!("{kind:?}") }; let flt_cc = match kind diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index 46a441488fa6a..68ff0b622c8fb 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -205,9 +205,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( // Find a way to reuse `immediate_const_vector` from `codegen_ssa` instead. let indexes = { use rustc_middle::mir::interpret::*; - let idx_const = match &idx.node { - Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0, - Operand::Copy(_) | Operand::Move(_) => unreachable!("{idx:?}"), + let idx_const = if let Some(const_) = idx.node.constant() { + crate::constant::eval_mir_constant(fx, const_).0 + } else { + unreachable!("{idx:?}") }; let idx_bytes = match idx_const { From 2457bb2cfbda090a0a688351ab9b480ead129879 Mon Sep 17 00:00:00 2001 From: beetrees Date: Thu, 12 Jun 2025 00:47:01 +0100 Subject: [PATCH 19/38] Apply ABI attributes on return types in `rustc_codegen_cranelift` --- src/abi/pass_mode.rs | 59 +++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 6d8614aca6932..9a0a5b510397a 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -1,6 +1,6 @@ //! Argument passing -use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose}; +use cranelift_codegen::ir::ArgumentPurpose; use rustc_abi::{Reg, RegKind}; use rustc_target::callconv::{ ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, @@ -32,13 +32,12 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam { AbiParam::new(clif_ty) } -fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -> AbiParam { +fn apply_attrs_to_abi_param(param: AbiParam, arg_attrs: ArgAttributes) -> AbiParam { match arg_attrs.arg_ext { - RustcArgExtension::None => {} - RustcArgExtension::Zext => param.extension = ArgumentExtension::Uext, - RustcArgExtension::Sext => param.extension = ArgumentExtension::Sext, + RustcArgExtension::None => param, + RustcArgExtension::Zext => param.uext(), + RustcArgExtension::Sext => param.sext(), } - param } fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> { @@ -82,7 +81,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { match self.mode { PassMode::Ignore => smallvec![], PassMode::Direct(attrs) => match self.layout.backend_repr { - BackendRepr::Scalar(scalar) => smallvec![apply_arg_attrs_to_abi_param( + BackendRepr::Scalar(scalar) => smallvec![apply_attrs_to_abi_param( AbiParam::new(scalar_to_clif_type(tcx, scalar)), attrs )], @@ -97,8 +96,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { let a = scalar_to_clif_type(tcx, a); let b = scalar_to_clif_type(tcx, b); smallvec![ - apply_arg_attrs_to_abi_param(AbiParam::new(a), attrs_a), - apply_arg_attrs_to_abi_param(AbiParam::new(b), attrs_b), + apply_attrs_to_abi_param(AbiParam::new(a), attrs_a), + apply_attrs_to_abi_param(AbiParam::new(b), attrs_b), ] } _ => unreachable!("{:?}", self.layout.backend_repr), @@ -112,19 +111,19 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { // Abi requires aligning struct size to pointer size let size = self.layout.size.align_to(tcx.data_layout.pointer_align.abi); let size = u32::try_from(size.bytes()).unwrap(); - smallvec![apply_arg_attrs_to_abi_param( + smallvec![apply_attrs_to_abi_param( AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),), attrs )] } else { - smallvec![apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)] + smallvec![apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)] } } PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { assert!(!on_stack); smallvec![ - apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), - apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), meta_attrs), + apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), + apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), meta_attrs), ] } } @@ -133,30 +132,46 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option, Vec) { match self.mode { PassMode::Ignore => (None, vec![]), - PassMode::Direct(_) => match self.layout.backend_repr { - BackendRepr::Scalar(scalar) => { - (None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar))]) - } + PassMode::Direct(attrs) => match self.layout.backend_repr { + BackendRepr::Scalar(scalar) => ( + None, + vec![apply_attrs_to_abi_param( + AbiParam::new(scalar_to_clif_type(tcx, scalar)), + attrs, + )], + ), BackendRepr::SimdVector { .. } => { let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout); - (None, vec![AbiParam::new(vector_ty)]) + (None, vec![apply_attrs_to_abi_param(AbiParam::new(vector_ty), attrs)]) } _ => unreachable!("{:?}", self.layout.backend_repr), }, - PassMode::Pair(_, _) => match self.layout.backend_repr { + PassMode::Pair(attrs_a, attrs_b) => match self.layout.backend_repr { BackendRepr::ScalarPair(a, b) => { let a = scalar_to_clif_type(tcx, a); let b = scalar_to_clif_type(tcx, b); - (None, vec![AbiParam::new(a), AbiParam::new(b)]) + ( + None, + vec![ + apply_attrs_to_abi_param(AbiParam::new(a), attrs_a), + apply_attrs_to_abi_param(AbiParam::new(b), attrs_b), + ], + ) } _ => unreachable!("{:?}", self.layout.backend_repr), }, PassMode::Cast { ref cast, .. } => { (None, cast_target_to_abi_params(cast).into_iter().collect()) } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack } => { + PassMode::Indirect { attrs, meta_attrs: None, on_stack } => { assert!(!on_stack); - (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![]) + ( + Some(apply_attrs_to_abi_param( + AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn), + attrs, + )), + vec![], + ) } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { unreachable!("unsized return value") From a9fd42e53627e53d32d1df6819e9958b7d3ee8f9 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Thu, 12 Jun 2025 13:44:19 +0200 Subject: [PATCH 20/38] intrinsics: rename min_align_of to align_of --- example/mini_core.rs | 4 ++-- example/mini_core_hello_world.rs | 7 ++----- src/intrinsics/mod.rs | 4 ++-- src/unsize.rs | 2 +- src/vtable.rs | 2 +- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/example/mini_core.rs b/example/mini_core.rs index 1dc799c0aeea8..012e4dbc3ec9e 100644 --- a/example/mini_core.rs +++ b/example/mini_core.rs @@ -644,9 +644,9 @@ pub mod intrinsics { #[rustc_intrinsic] pub unsafe fn size_of_val(val: *const T) -> usize; #[rustc_intrinsic] - pub fn min_align_of() -> usize; + pub fn align_of() -> usize; #[rustc_intrinsic] - pub unsafe fn min_align_of_val(val: *const T) -> usize; + pub unsafe fn align_of_val(val: *const T) -> usize; #[rustc_intrinsic] pub unsafe fn copy(src: *const T, dst: *mut T, count: usize); #[rustc_intrinsic] diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index 93ca2e0e42188..1499f948deb3c 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -204,11 +204,8 @@ fn main() { assert_eq!(intrinsics::size_of_val(a) as u8, 16); assert_eq!(intrinsics::size_of_val(&0u32) as u8, 4); - assert_eq!(intrinsics::min_align_of::() as u8, 2); - assert_eq!( - intrinsics::min_align_of_val(&a) as u8, - intrinsics::min_align_of::<&str>() as u8 - ); + assert_eq!(intrinsics::align_of::() as u8, 2); + assert_eq!(intrinsics::align_of_val(&a) as u8, intrinsics::align_of::<&str>() as u8); assert!(!intrinsics::needs_drop::()); assert!(!intrinsics::needs_drop::<[u8]>()); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 1d1cf884e48b2..df5748c34d110 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -586,7 +586,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let (size, _align) = crate::unsize::size_and_align_of(fx, layout, meta); ret.write_cvalue(fx, CValue::by_val(size, usize_layout)); } - sym::min_align_of_val => { + sym::align_of_val => { intrinsic_args!(fx, args => (ptr); intrinsic); let layout = fx.layout_of(generic_args.type_at(0)); @@ -613,7 +613,7 @@ fn codegen_regular_intrinsic_call<'tcx>( intrinsic_args!(fx, args => (vtable); intrinsic); let vtable = vtable.load_scalar(fx); - let align = crate::vtable::min_align_of_obj(fx, vtable); + let align = crate::vtable::align_of_obj(fx, vtable); ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); } diff --git a/src/unsize.rs b/src/unsize.rs index 662546e499988..df60b05c4636c 100644 --- a/src/unsize.rs +++ b/src/unsize.rs @@ -212,7 +212,7 @@ pub(crate) fn size_and_align_of<'tcx>( // load size/align from vtable ( crate::vtable::size_of_obj(fx, info.unwrap()), - crate::vtable::min_align_of_obj(fx, info.unwrap()), + crate::vtable::align_of_obj(fx, info.unwrap()), ) } ty::Slice(_) | ty::Str => { diff --git a/src/vtable.rs b/src/vtable.rs index 05a8e3c334216..1fae56949bc05 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -31,7 +31,7 @@ pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Val ) } -pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value { +pub(crate) fn align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value { let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize; fx.bcx.ins().load( fx.pointer_type, From 9594d67bcf88c82546f8fae1382a06a239cddbe9 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 12 Jun 2025 16:48:29 +0000 Subject: [PATCH 21/38] Rustup to rustc 1.89.0-nightly (e703dff8f 2025-06-11) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 1e867bd76580a..fc8f892fb1e09 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-08" +channel = "nightly-2025-06-12" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From dcc48749e21d24b45b22f37734f0058138d71efc Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Wed, 7 May 2025 20:02:29 +0200 Subject: [PATCH 22/38] add `extern "custom"` functions --- src/abi/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index fe5b220117f3c..4c6fd90781543 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -51,6 +51,11 @@ pub(crate) fn conv_to_call_conv( CanonAbi::Rust | CanonAbi::C => default_call_conv, CanonAbi::RustCold => CallConv::Cold, + // Functions with this calling convention can only be called from assembly, but it is + // possible to declare an `extern "custom"` block, so the backend still needs a calling + // convention for declaring foreign functions. + CanonAbi::Custom => default_call_conv, + CanonAbi::X86(x86_call) => match x86_call { X86Call::SysV64 => CallConv::SystemV, X86Call::Win64 => CallConv::WindowsFastcall, From a6ecde32b94f4155587c97d69f07c3d3cd174203 Mon Sep 17 00:00:00 2001 From: mejrs <59372212+mejrs@users.noreply.github.com> Date: Fri, 13 Jun 2025 01:16:36 +0200 Subject: [PATCH 23/38] Unimplement unsized_locals --- example/arbitrary_self_types_pointers_and_wrappers.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/example/arbitrary_self_types_pointers_and_wrappers.rs b/example/arbitrary_self_types_pointers_and_wrappers.rs index 5479b0c617bfb..f0d1f6e221572 100644 --- a/example/arbitrary_self_types_pointers_and_wrappers.rs +++ b/example/arbitrary_self_types_pointers_and_wrappers.rs @@ -32,10 +32,6 @@ impl, U> CoerceUnsized> for Wrapper {} impl, U> DispatchFromDyn> for Wrapper {} trait Trait { - // This method isn't object-safe yet. Unsized by-value `self` is object-safe (but not callable - // without unsized_locals), but wrappers around `Self` currently are not. - // FIXME (mikeyhew) uncomment this when unsized rvalues object-safety is implemented - // fn wrapper(self: Wrapper) -> i32; fn ptr_wrapper(self: Ptr>) -> i32; fn wrapper_ptr(self: Wrapper>) -> i32; fn wrapper_ptr_wrapper(self: Wrapper>>) -> i32; From 600a8a7f63f2935ac4ad67637c03df83b99c1eca Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 14 Jun 2025 09:42:41 +0000 Subject: [PATCH 24/38] Rustup to rustc 1.89.0-nightly (8da623945 2025-06-13) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index fc8f892fb1e09..5114d3686a43d 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-12" +channel = "nightly-2025-06-14" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From 3e59f14fb286859e01f467e17aa3fb251802e670 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 15 Jun 2025 11:19:04 +0000 Subject: [PATCH 25/38] Update to abi-cafe 1.0 --- build_system/abi_cafe.rs | 4 +- .../0002-abi-cafe-Disable-broken-tests.patch | 68 +++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index 674acfbd3097c..bee59b2b95607 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -6,8 +6,8 @@ use crate::{CodegenBackend, SysrootKind, build_sysroot}; static ABI_CAFE_REPO: GitRepo = GitRepo::github( "Gankra", "abi-cafe", - "f1220cfd13b57f5c0082c26529163865ee25e115", - "fe93a9acd461425d", + "49efdca14e9d48c34e50552d1f620615a87637bb", + "21fed1b4ca4cd439", "abi-cafe", ); diff --git a/patches/0002-abi-cafe-Disable-broken-tests.patch b/patches/0002-abi-cafe-Disable-broken-tests.patch index 01b6a990b7202..708bef0edcf8f 100644 --- a/patches/0002-abi-cafe-Disable-broken-tests.patch +++ b/patches/0002-abi-cafe-Disable-broken-tests.patch @@ -4,7 +4,7 @@ Date: Tue, 9 Jul 2024 11:25:14 +0000 Subject: [PATCH] Disable broken tests --- - src/report.rs | 36 ++++++++++++++++++++++++++++++++++++ + src/harness/report.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/toolchains/rust.rs b/src/toolchains/rust.rs @@ -14,56 +14,56 @@ index 0c50f7a..bfde2b1 100644 @@ -83,6 +83,7 @@ impl Toolchain for RustcToolchain { .arg(out_dir) .arg("--target") - .arg(built_info::TARGET) + .arg(&self.platform_info.target) + .arg("-g") .arg(format!("-Cmetadata={lib_name}")) .arg(src_path); if let Some(codegen_backend) = &self.codegen_backend { -diff --git a/src/report.rs b/src/report.rs +diff --git a/src/harness/report.rs b/src/harness/report.rs index 958ab43..dcf1044 100644 ---- a/src/report.rs -+++ b/src/report.rs +--- a/src/harness/report.rs ++++ b/src/harness/report.rs @@ -48,6 +48,40 @@ pub fn get_test_rules(test: &TestKey, caller: &dyn Toolchain, callee: &dyn Toolc - // - // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES + // + // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES -+ if cfg!(all(target_arch = "aarch64", target_os = "linux")) { -+ if test.test == "F32Array" && test.options.convention == CallingConvention::C { -+ result.check = Busted(Check); ++ if cfg!(all(target_arch = "aarch64", target_os = "linux")) { ++ if key.test == "F32Array" && key.options.convention == CallingConvention::C { ++ result.check = Busted(Check); ++ } + } -+ } + -+ if cfg!(all(target_arch = "aarch64", target_os = "macos")) { -+ if test.test == "SingleVariantUnion" && test.options.convention == CallingConvention::C && test.options.repr == LangRepr::C { -+ result.check = Busted(Check); -+ } ++ if cfg!(all(target_arch = "aarch64", target_os = "macos")) { ++ if key.test == "SingleVariantUnion" && key.options.convention == CallingConvention::C && key.options.repr == LangRepr::C { ++ result.check = Busted(Check); ++ } + -+ if test.test == "OptionU128" && test.caller == "rustc" && test.options.convention == CallingConvention::Rust && test.options.repr == LangRepr::C { -+ result.check = Busted(Run); -+ } ++ if key.test == "OptionU128" && key.caller == "rustc" && key.options.convention == CallingConvention::Rust && key.options.repr == LangRepr::C { ++ result.check = Busted(Run); ++ } + -+ if test.test == "OptionU128" && test.caller == "cgclif" && test.options.convention == CallingConvention::Rust && test.options.repr == LangRepr::C { -+ result.check = Busted(Check); ++ if key.test == "OptionU128" && key.caller == "cgclif" && key.options.convention == CallingConvention::Rust && key.options.repr == LangRepr::C { ++ result.check = Busted(Check); ++ } + } -+ } + -+ if cfg!(all(target_arch = "x86_64", windows)) { -+ if test.test == "simple" && test.options.convention == CallingConvention::Rust { -+ result.check = Busted(Check); -+ } ++ if cfg!(all(target_arch = "x86_64", windows)) { ++ if key.test == "simple" && key.options.convention == CallingConvention::Rust { ++ result.check = Busted(Check); ++ } + -+ if test.test == "simple" && test.options.convention == CallingConvention::Rust && test.caller == "rustc" { -+ result.check = Busted(Run); ++ if key.test == "simple" && key.options.convention == CallingConvention::Rust && key.caller == "rustc" { ++ result.check = Busted(Run); ++ } + } -+ } + -+ if test.test == "f16" || test.test == "f128" { -+ result.run = Skip; -+ } ++ if key.test == "f16" || key.test == "f128" { ++ result.run = Skip; ++ } + - // END OF VENDOR RESERVED AREA - // - // + // END OF VENDOR RESERVED AREA + // + // -- 2.34.1 From 8a396193432f4a9cb25ab76f6fdeefe08484edf8 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 15 Jun 2025 12:09:50 +0000 Subject: [PATCH 26/38] Use the new abi-cafe-rules.toml support --- build_system/abi_cafe.rs | 2 + .../0002-abi-cafe-Disable-broken-tests.patch | 69 ------------------- patches/0002-abi-cafe-Enable-debuginfo.patch | 24 +++++++ scripts/abi-cafe-rules.toml | 17 +++++ 4 files changed, 43 insertions(+), 69 deletions(-) delete mode 100644 patches/0002-abi-cafe-Disable-broken-tests.patch create mode 100644 patches/0002-abi-cafe-Enable-debuginfo.patch create mode 100644 scripts/abi-cafe-rules.toml diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index bee59b2b95607..8f8884d2ac942 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -46,6 +46,8 @@ pub(crate) fn run( let mut cmd = ABI_CAFE.run(bootstrap_host_compiler, dirs); cmd.arg("--"); + cmd.arg("--rules").arg(dirs.source_dir.join("scripts/abi-cafe-rules.toml")); + // stdcall, vectorcall and such don't work yet cmd.arg("--conventions").arg("c").arg("--conventions").arg("rust"); diff --git a/patches/0002-abi-cafe-Disable-broken-tests.patch b/patches/0002-abi-cafe-Disable-broken-tests.patch deleted file mode 100644 index 708bef0edcf8f..0000000000000 --- a/patches/0002-abi-cafe-Disable-broken-tests.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 236df390f3bc4ed69c26f4d51d584bea246da886 Mon Sep 17 00:00:00 2001 -From: bjorn3 <17426603+bjorn3@users.noreply.github.com> -Date: Tue, 9 Jul 2024 11:25:14 +0000 -Subject: [PATCH] Disable broken tests - ---- - src/harness/report.rs | 36 ++++++++++++++++++++++++++++++++++++ - 1 file changed, 36 insertions(+) - -diff --git a/src/toolchains/rust.rs b/src/toolchains/rust.rs -index 0c50f7a..bfde2b1 100644 ---- a/src/toolchains/rust.rs -+++ b/src/toolchains/rust.rs -@@ -83,6 +83,7 @@ impl Toolchain for RustcToolchain { - .arg(out_dir) - .arg("--target") - .arg(&self.platform_info.target) -+ .arg("-g") - .arg(format!("-Cmetadata={lib_name}")) - .arg(src_path); - if let Some(codegen_backend) = &self.codegen_backend { -diff --git a/src/harness/report.rs b/src/harness/report.rs -index 958ab43..dcf1044 100644 ---- a/src/harness/report.rs -+++ b/src/harness/report.rs -@@ -48,6 +48,40 @@ pub fn get_test_rules(test: &TestKey, caller: &dyn Toolchain, callee: &dyn Toolc - // - // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES - -+ if cfg!(all(target_arch = "aarch64", target_os = "linux")) { -+ if key.test == "F32Array" && key.options.convention == CallingConvention::C { -+ result.check = Busted(Check); -+ } -+ } -+ -+ if cfg!(all(target_arch = "aarch64", target_os = "macos")) { -+ if key.test == "SingleVariantUnion" && key.options.convention == CallingConvention::C && key.options.repr == LangRepr::C { -+ result.check = Busted(Check); -+ } -+ -+ if key.test == "OptionU128" && key.caller == "rustc" && key.options.convention == CallingConvention::Rust && key.options.repr == LangRepr::C { -+ result.check = Busted(Run); -+ } -+ -+ if key.test == "OptionU128" && key.caller == "cgclif" && key.options.convention == CallingConvention::Rust && key.options.repr == LangRepr::C { -+ result.check = Busted(Check); -+ } -+ } -+ -+ if cfg!(all(target_arch = "x86_64", windows)) { -+ if key.test == "simple" && key.options.convention == CallingConvention::Rust { -+ result.check = Busted(Check); -+ } -+ -+ if key.test == "simple" && key.options.convention == CallingConvention::Rust && key.caller == "rustc" { -+ result.check = Busted(Run); -+ } -+ } -+ -+ if key.test == "f16" || key.test == "f128" { -+ result.run = Skip; -+ } -+ - // END OF VENDOR RESERVED AREA - // - // --- -2.34.1 - diff --git a/patches/0002-abi-cafe-Enable-debuginfo.patch b/patches/0002-abi-cafe-Enable-debuginfo.patch new file mode 100644 index 0000000000000..6f5c588f2b2f6 --- /dev/null +++ b/patches/0002-abi-cafe-Enable-debuginfo.patch @@ -0,0 +1,24 @@ +From 236df390f3bc4ed69c26f4d51d584bea246da886 Mon Sep 17 00:00:00 2001 +From: bjorn3 <17426603+bjorn3@users.noreply.github.com> +Date: Tue, 9 Jul 2024 11:25:14 +0000 +Subject: [PATCH] Enable debuginfo + +--- + src/toolchains/rust.rs | 1 + + 1 file changed, 1 insertions(+) + +diff --git a/src/toolchains/rust.rs b/src/toolchains/rust.rs +index 0c50f7a..bfde2b1 100644 +--- a/src/toolchains/rust.rs ++++ b/src/toolchains/rust.rs +@@ -83,6 +83,7 @@ impl Toolchain for RustcToolchain { + .arg(out_dir) + .arg("--target") + .arg(&self.platform_info.target) ++ .arg("-g") + .arg(format!("-Cmetadata={lib_name}")) + .arg(src_path); + if let Some(codegen_backend) = &self.codegen_backend { +-- +2.34.1 + diff --git a/scripts/abi-cafe-rules.toml b/scripts/abi-cafe-rules.toml new file mode 100644 index 0000000000000..54f9445c8e524 --- /dev/null +++ b/scripts/abi-cafe-rules.toml @@ -0,0 +1,17 @@ +[target.'cfg(all(target_arch = "aarch64", target_os = "linux"))'] +'F32Array::conv_c'.busted = "check" + +[target.'cfg(all(target_arch = "aarch64", target_os = "macos"))'] +'SingleVariantUnion::conv_c::repr_c'.busted = "check" +'OptionU128::conv_rust::repr_c::rustc_caller'.busted = "run" +'OptionU128::conv_rust::repr_c::cgclif_caller'.busted = "check" + +[target.'cfg(all(target_arch = "x86_64", windows))'] +'simple::conv_rust'.busted = "check" +'simple::conv_rust::rustc_caller'.busted = "run" + +[target.'*'.'f16'] +run = "skip" + +[target.'*'.'f128'] +run = "skip" From 1ade48fde9faeb4e161119619aaa4f940acab79b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 16 Jun 2025 08:15:58 +0000 Subject: [PATCH 27/38] Rustup to rustc 1.89.0-nightly (586ad391f 2025-06-15) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index 5114d3686a43d..d92c80a3d9342 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-14" +channel = "nightly-2025-06-16" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From b7cfe2f4db9e7740f6302a1627d1c087054e64b4 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 16 Jun 2025 08:23:24 +0000 Subject: [PATCH 28/38] Use the new --debug flag of abi-cafe As opposed to patching abi-cafe itself. --- build_system/abi_cafe.rs | 6 +++-- patches/0002-abi-cafe-Enable-debuginfo.patch | 24 -------------------- 2 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 patches/0002-abi-cafe-Enable-debuginfo.patch diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index 8f8884d2ac942..43025137bc6bb 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -6,8 +6,8 @@ use crate::{CodegenBackend, SysrootKind, build_sysroot}; static ABI_CAFE_REPO: GitRepo = GitRepo::github( "Gankra", "abi-cafe", - "49efdca14e9d48c34e50552d1f620615a87637bb", - "21fed1b4ca4cd439", + "94d38030419eb00a1ba80e5e2b4d763dcee58db4", + "6efb4457893c8670", "abi-cafe", ); @@ -46,6 +46,8 @@ pub(crate) fn run( let mut cmd = ABI_CAFE.run(bootstrap_host_compiler, dirs); cmd.arg("--"); + cmd.arg("--debug"); + cmd.arg("--rules").arg(dirs.source_dir.join("scripts/abi-cafe-rules.toml")); // stdcall, vectorcall and such don't work yet diff --git a/patches/0002-abi-cafe-Enable-debuginfo.patch b/patches/0002-abi-cafe-Enable-debuginfo.patch deleted file mode 100644 index 6f5c588f2b2f6..0000000000000 --- a/patches/0002-abi-cafe-Enable-debuginfo.patch +++ /dev/null @@ -1,24 +0,0 @@ -From 236df390f3bc4ed69c26f4d51d584bea246da886 Mon Sep 17 00:00:00 2001 -From: bjorn3 <17426603+bjorn3@users.noreply.github.com> -Date: Tue, 9 Jul 2024 11:25:14 +0000 -Subject: [PATCH] Enable debuginfo - ---- - src/toolchains/rust.rs | 1 + - 1 file changed, 1 insertions(+) - -diff --git a/src/toolchains/rust.rs b/src/toolchains/rust.rs -index 0c50f7a..bfde2b1 100644 ---- a/src/toolchains/rust.rs -+++ b/src/toolchains/rust.rs -@@ -83,6 +83,7 @@ impl Toolchain for RustcToolchain { - .arg(out_dir) - .arg("--target") - .arg(&self.platform_info.target) -+ .arg("-g") - .arg(format!("-Cmetadata={lib_name}")) - .arg(src_path); - if let Some(codegen_backend) = &self.codegen_backend { --- -2.34.1 - From e3916036f927e2bfc2ff356df08ef2e1d1b29378 Mon Sep 17 00:00:00 2001 From: beetrees Date: Thu, 3 Apr 2025 23:53:06 +0100 Subject: [PATCH 29/38] Fix RISC-V C function ABI when passing/returning structs containing floats --- src/abi/pass_mode.rs | 66 ++++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 9a0a5b510397a..cd0afee0cfb2f 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -40,7 +40,18 @@ fn apply_attrs_to_abi_param(param: AbiParam, arg_attrs: ArgAttributes) -> AbiPar } } -fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> { +fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[(Size, AbiParam); 2]> { + if let Some(offset_from_start) = cast.rest_offset { + assert!(cast.prefix[1..].iter().all(|p| p.is_none())); + assert_eq!(cast.rest.unit.size, cast.rest.total); + let first = cast.prefix[0].unwrap(); + let second = cast.rest.unit; + return smallvec![ + (Size::ZERO, reg_to_abi_param(first)), + (offset_from_start, reg_to_abi_param(second)) + ]; + } + let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { (0, 0) } else { @@ -55,25 +66,32 @@ fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> { // different types in Cranelift IR. Instead a single array of primitive types is used. // Create list of fields in the main structure - let mut args = cast + let args = cast .prefix .iter() .flatten() .map(|®| reg_to_abi_param(reg)) - .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit))) - .collect::>(); + .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit))); + + let mut res = SmallVec::new(); + let mut offset = Size::ZERO; + + for arg in args { + res.push((offset, arg)); + offset += Size::from_bytes(arg.value_type.bytes()); + } // Append final integer if rem_bytes != 0 { // Only integers can be really split further. assert_eq!(cast.rest.unit.kind, RegKind::Integer); - args.push(reg_to_abi_param(Reg { - kind: RegKind::Integer, - size: Size::from_bytes(rem_bytes), - })); + res.push(( + offset, + reg_to_abi_param(Reg { kind: RegKind::Integer, size: Size::from_bytes(rem_bytes) }), + )); } - args + res } impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { @@ -104,7 +122,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { }, PassMode::Cast { ref cast, pad_i32 } => { assert!(!pad_i32, "padding support not yet implemented"); - cast_target_to_abi_params(cast) + cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect() } PassMode::Indirect { attrs, meta_attrs: None, on_stack } => { if on_stack { @@ -160,9 +178,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.backend_repr), }, - PassMode::Cast { ref cast, .. } => { - (None, cast_target_to_abi_params(cast).into_iter().collect()) - } + PassMode::Cast { ref cast, .. } => ( + None, + cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(), + ), PassMode::Indirect { attrs, meta_attrs: None, on_stack } => { assert!(!on_stack); ( @@ -187,12 +206,14 @@ pub(super) fn to_casted_value<'tcx>( ) -> SmallVec<[Value; 2]> { let (ptr, meta) = arg.force_stack(fx); assert!(meta.is_none()); - let mut offset = 0; cast_target_to_abi_params(cast) .into_iter() - .map(|param| { - let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new()); - offset += i64::from(param.value_type.bytes()); + .map(|(offset, param)| { + let val = ptr.offset_i64(fx, offset.bytes() as i64).load( + fx, + param.value_type, + MemFlags::new(), + ); val }) .collect() @@ -205,7 +226,7 @@ pub(super) fn from_casted_value<'tcx>( cast: &CastTarget, ) -> CValue<'tcx> { let abi_params = cast_target_to_abi_params(cast); - let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum(); + let abi_param_size: u32 = abi_params.iter().map(|(_, param)| param.value_type.bytes()).sum(); let layout_size = u32::try_from(layout.size.bytes()).unwrap(); let ptr = fx.create_stack_slot( // Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`. @@ -214,16 +235,13 @@ pub(super) fn from_casted_value<'tcx>( std::cmp::max(abi_param_size, layout_size), u32::try_from(layout.align.abi.bytes()).unwrap(), ); - let mut offset = 0; let mut block_params_iter = block_params.iter().copied(); - for param in abi_params { - let val = ptr.offset_i64(fx, offset).store( + for (offset, _) in abi_params { + ptr.offset_i64(fx, offset.bytes() as i64).store( fx, block_params_iter.next().unwrap(), MemFlags::new(), - ); - offset += i64::from(param.value_type.bytes()); - val + ) } assert_eq!(block_params_iter.next(), None, "Leftover block param"); CValue::by_ref(ptr, layout) From db5a427b3104a8597cca338ec0da3ab0243b7864 Mon Sep 17 00:00:00 2001 From: Daniel Paoliello Date: Thu, 15 May 2025 16:38:46 -0700 Subject: [PATCH 30/38] Change __rust_no_alloc_shim_is_unstable to be a function --- src/allocator.rs | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/allocator.rs b/src/allocator.rs index 9cff8a84db3d9..ffb932a3c38e6 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -1,6 +1,7 @@ //! Allocator shim // Adapted from rustc +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use rustc_ast::expand::allocator::{ ALLOCATOR_METHODS, AllocatorKind, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, alloc_error_handler_name, default_fn_name, global_fn_name, @@ -97,16 +98,31 @@ fn codegen_inner( data.define(Box::new([val])); module.define_data(data_id, &data).unwrap(); - let data_id = module - .declare_data( - &mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), - Linkage::Export, - false, - false, - ) - .unwrap(); - let mut data = DataDescription::new(); - data.set_align(1); - data.define(Box::new([0])); - module.define_data(data_id, &data).unwrap(); + { + let sig = Signature { + call_conv: module.target_config().default_call_conv, + params: vec![], + returns: vec![], + }; + let func_id = module + .declare_function( + &mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), + Linkage::Export, + &sig, + ) + .unwrap(); + + let mut ctx = Context::new(); + ctx.func.signature = sig; + let mut func_ctx = FunctionBuilderContext::new(); + let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx); + + let block = bcx.create_block(); + bcx.switch_to_block(block); + bcx.ins().return_(&[]); + bcx.seal_all_blocks(); + bcx.finalize(); + + module.define_function(func_id, &mut ctx).unwrap(); + } } From cfa4c8f473cbe5af1a6665add451ee0ec702f898 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 5 Mar 2025 04:31:12 +0000 Subject: [PATCH 31/38] cranelift/gcc: `{Meta,Pointee,}Sized` in minicore As in many previous commits, adding the new traits to minicore, but this time for cranelift and gcc. --- example/mini_core.rs | 66 ++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/example/mini_core.rs b/example/mini_core.rs index 012e4dbc3ec9e..524ebde1c743f 100644 --- a/example/mini_core.rs +++ b/example/mini_core.rs @@ -14,8 +14,14 @@ #![no_core] #![allow(dead_code, internal_features, ambiguous_wide_pointer_comparisons)] +#[lang = "pointee_sized"] +pub trait PointeeSized {} + +#[lang = "meta_sized"] +pub trait MetaSized: PointeeSized {} + #[lang = "sized"] -pub trait Sized {} +pub trait Sized: MetaSized {} #[lang = "destruct"] pub trait Destruct {} @@ -24,35 +30,35 @@ pub trait Destruct {} pub trait Tuple {} #[lang = "unsize"] -pub trait Unsize {} +pub trait Unsize: PointeeSized {} #[lang = "coerce_unsized"] pub trait CoerceUnsized {} -impl<'a, 'b: 'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<&'a U> for &'b T {} -impl<'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<&'a mut U> for &'a mut T {} -impl, U: ?Sized> CoerceUnsized<*const U> for *const T {} -impl, U: ?Sized> CoerceUnsized<*mut U> for *mut T {} +impl<'a, 'b: 'a, T: PointeeSized + Unsize, U: PointeeSized> CoerceUnsized<&'a U> for &'b T {} +impl<'a, T: PointeeSized + Unsize, U: PointeeSized> CoerceUnsized<&'a mut U> for &'a mut T {} +impl, U: PointeeSized> CoerceUnsized<*const U> for *const T {} +impl, U: PointeeSized> CoerceUnsized<*mut U> for *mut T {} #[lang = "dispatch_from_dyn"] pub trait DispatchFromDyn {} // &T -> &U -impl<'a, T: ?Sized + Unsize, U: ?Sized> DispatchFromDyn<&'a U> for &'a T {} +impl<'a, T: PointeeSized + Unsize, U: PointeeSized> DispatchFromDyn<&'a U> for &'a T {} // &mut T -> &mut U -impl<'a, T: ?Sized + Unsize, U: ?Sized> DispatchFromDyn<&'a mut U> for &'a mut T {} +impl<'a, T: PointeeSized + Unsize, U: PointeeSized> DispatchFromDyn<&'a mut U> for &'a mut T {} // *const T -> *const U -impl, U: ?Sized> DispatchFromDyn<*const U> for *const T {} +impl, U: PointeeSized> DispatchFromDyn<*const U> for *const T {} // *mut T -> *mut U -impl, U: ?Sized> DispatchFromDyn<*mut U> for *mut T {} -impl, U: ?Sized> DispatchFromDyn> for Box {} +impl, U: PointeeSized> DispatchFromDyn<*mut U> for *mut T {} +impl, U: MetaSized> DispatchFromDyn> for Box {} #[lang = "legacy_receiver"] pub trait LegacyReceiver {} -impl LegacyReceiver for &T {} -impl LegacyReceiver for &mut T {} -impl LegacyReceiver for Box {} +impl LegacyReceiver for &T {} +impl LegacyReceiver for &mut T {} +impl LegacyReceiver for Box {} #[lang = "copy"] pub trait Copy {} @@ -74,9 +80,9 @@ impl Copy for isize {} impl Copy for f32 {} impl Copy for f64 {} impl Copy for char {} -impl<'a, T: ?Sized> Copy for &'a T {} -impl Copy for *const T {} -impl Copy for *mut T {} +impl<'a, T: PointeeSized> Copy for &'a T {} +impl Copy for *const T {} +impl Copy for *mut T {} impl Copy for Option {} #[lang = "sync"] @@ -94,17 +100,17 @@ unsafe impl Sync for i32 {} unsafe impl Sync for isize {} unsafe impl Sync for char {} unsafe impl Sync for f32 {} -unsafe impl<'a, T: ?Sized> Sync for &'a T {} +unsafe impl<'a, T: PointeeSized> Sync for &'a T {} unsafe impl Sync for [T; N] {} #[lang = "freeze"] unsafe auto trait Freeze {} -unsafe impl Freeze for PhantomData {} -unsafe impl Freeze for *const T {} -unsafe impl Freeze for *mut T {} -unsafe impl Freeze for &T {} -unsafe impl Freeze for &mut T {} +unsafe impl Freeze for PhantomData {} +unsafe impl Freeze for *const T {} +unsafe impl Freeze for *mut T {} +unsafe impl Freeze for &T {} +unsafe impl Freeze for &mut T {} #[lang = "structural_peq"] pub trait StructuralPartialEq {} @@ -443,7 +449,7 @@ pub enum Option { pub use Option::*; #[lang = "phantom_data"] -pub struct PhantomData; +pub struct PhantomData; #[lang = "fn_once"] #[rustc_paren_sugar] @@ -564,18 +570,18 @@ pub trait Deref { #[repr(transparent)] #[rustc_layout_scalar_valid_range_start(1)] #[rustc_nonnull_optimization_guaranteed] -pub struct NonNull(pub *const T); +pub struct NonNull(pub *const T); -impl CoerceUnsized> for NonNull where T: Unsize {} -impl DispatchFromDyn> for NonNull where T: Unsize {} +impl CoerceUnsized> for NonNull where T: Unsize {} +impl DispatchFromDyn> for NonNull where T: Unsize {} -pub struct Unique { +pub struct Unique { pub pointer: NonNull, pub _marker: PhantomData, } -impl CoerceUnsized> for Unique where T: Unsize {} -impl DispatchFromDyn> for Unique where T: Unsize {} +impl CoerceUnsized> for Unique where T: Unsize {} +impl DispatchFromDyn> for Unique where T: Unsize {} #[lang = "global_alloc_ty"] pub struct Global; From da831be4034d11297ab02740d52ae454a60dcc6f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 17 Jun 2025 12:00:57 +0000 Subject: [PATCH 32/38] Rustup to rustc 1.89.0-nightly (45acf54ee 2025-06-16) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index d92c80a3d9342..a9ca0f23d8bea 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-16" +channel = "nightly-2025-06-17" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From 8dea73acfd76a38173975ff1e60b7770af677768 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 19 Jun 2025 09:28:38 +0000 Subject: [PATCH 33/38] Rustup to rustc 1.89.0-nightly (c68340350 2025-06-18) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index a9ca0f23d8bea..f09a2628777da 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-17" +channel = "nightly-2025-06-19" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From 3719abbb33c7925e29a7d5c92a5c33e1ef0cb3ca Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 7 Jun 2025 18:07:43 +0000 Subject: [PATCH 34/38] Update to Cranelift 0.121 --- Cargo.lock | 84 ++++++++++++++++++++++++++++++++---------------------- Cargo.toml | 24 ++++++++-------- 2 files changed, 62 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a906bec8b7e23..b893a2be9a2cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,42 +43,42 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-assembler-x64" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff8e35182c7372df00447cb90a04e584e032c42b9b9b6e8c50ddaaf0d7900d5" +checksum = "f6f53499803b1607b6ee0ba0de4ba036e6da700c2e489fe8f9d0f683d0b84d31" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14220f9c2698015c3b94dc6b84ae045c1c45509ddc406e43c6139252757fdb7a" +checksum = "1aadaa5bc8430d0e7bb999459369bedd0e5816ad4a82a0e20748341c4e333eda" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d372ef2777ceefd75829e1390211ac240e9196bc60699218f7ea2419038288ee" +checksum = "2005fda2fc52a2dbce58229b4fb4483b70cbc806ba8ecc11b3f050c1a2d26cac" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56323783e423818fa89ce8078e90a3913d2a6e0810399bfce8ebd7ee87baa81f" +checksum = "56935e02452ca1249d39ad5c45a96304d0b4300a158a391fd113451e0cd4483d" [[package]] name = "cranelift-codegen" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74ffb780aab6186c6e9ba26519654b1ac55a09c0a866f6088a4efbbd84da68ed" +checksum = "62612786bf00e10999f50217d6f455d02b31591155881a45a903d1a95d1a4043" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -97,13 +97,14 @@ dependencies = [ "serde", "smallvec", "target-lexicon", + "wasmtime-math", ] [[package]] name = "cranelift-codegen-meta" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c23ef13814d3b39c869650d5961128cbbecad83fbdff4e6836a03ecf6862d7ed" +checksum = "07bae789df91ef236079733af9df11d852256c64af196f0bc6471ea0f5f301be" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -112,33 +113,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f623300657679f847803ce80811454bfff89cea4f6bf684be5c468d4a73631" +checksum = "1be319616d36527782558a8312508757815f64deb19b094c7b8f4337229a9bc6" [[package]] name = "cranelift-control" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4168af69989aa6b91fab46799ed4df6096f3209f4a6c8fb4358f49c60188f" +checksum = "8810ee1ab5e9bd5cff4c0c8d240e2009cb5c2b79888fde1d5256d605712314b7" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6fa9bae1c8de26d71ac2162f069447610fd91e7780cb480ee0d76ac81eabb8" +checksum = "086452c97cfbe116bf17dbe622dc5fdf2ea97299c7d4ce42460f284387c9928a" dependencies = [ "cranelift-bitset", ] [[package]] name = "cranelift-frontend" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8219205608aa0b0e6769b580284a7e055c7e0c323c1041cde7ca078add3e412" +checksum = "4c27947010ab759330f252610c17a8cd64d123358be4f33164233d04fcd77b80" dependencies = [ "cranelift-codegen", "log", @@ -148,15 +149,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "588d0c5964f10860b04043e55aab26d7f7a206b0fd4f10c5260e8aa5773832bd" +checksum = "ec67bfb8bd55b1e9760eb9f5186dca8d81bd4d86110f8d5af01154a044c91802" [[package]] name = "cranelift-jit" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56bd917ddc524f84f4066f954062875bdfc0dffea068ee94e906d98de5ac7c33" +checksum = "d67cdfc447f2abdb46bb30a6582cce189539c3c051c1d5330692376e1400edff" dependencies = [ "anyhow", "cranelift-codegen", @@ -174,9 +175,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a03c057d8a992e06596c871341e446af43ff9224f941e5b8adea39137a5391" +checksum = "e4597eaa52bca1ed111986c7a7f70cdbe192f83d271d627201365078e37b7e84" dependencies = [ "anyhow", "cranelift-codegen", @@ -185,9 +186,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed3c94cb97b14f92b6a94a1d45ef8c851f6a2ad9114e5d91d233f7da638fed" +checksum = "75a9b63edea46e013fce459c46e500462cb03a0490fdd9c18fe42b1dd7b93aa1" dependencies = [ "cranelift-codegen", "libc", @@ -196,9 +197,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64dacef362a69375a604f6636e5e9a174fb96dba3b273646fcd9fa85c1d0997" +checksum = "ce706f0166d5b7f31693dff521e87cb9858e12adf22ffcde93c4a2826f8f04a9" dependencies = [ "anyhow", "cranelift-codegen", @@ -211,9 +212,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.120.0" +version = "0.121.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85256fac1519a7d25a040c1d850fba67478f3f021ad5fdf738ba4425ee862dbf" +checksum = "7d5870e266df8237b56cc98b04f5739c228565c92dd629ec6c66efa87271a158" [[package]] name = "crc32fast" @@ -288,6 +289,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "log" version = "0.4.22" @@ -446,9 +453,9 @@ checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "wasmtime-jit-icache-coherence" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175e924dbc944c185808466d1e90b5a7feb610f3b9abdfe26f8ee25fd1086d1c" +checksum = "2eedc0324e37cf39b049f4dca0c30997eaab49f09006d5f4c1994e64e7b7dba8" dependencies = [ "anyhow", "cfg-if", @@ -456,6 +463,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "wasmtime-math" +version = "34.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd35fae4cf51d2b4a9bd2ef04b0eb309fa1849cab6a6ab5ac27cbd054ea284d" +dependencies = [ + "libm", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 94fcbd0a5023d..9066e4dbbb528 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,12 +8,12 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.120.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } -cranelift-frontend = { version = "0.120.0" } -cranelift-module = { version = "0.120.0" } -cranelift-native = { version = "0.120.0" } -cranelift-jit = { version = "0.120.0", optional = true } -cranelift-object = { version = "0.120.0" } +cranelift-codegen = { version = "0.121.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } +cranelift-frontend = { version = "0.121.0" } +cranelift-module = { version = "0.121.0" } +cranelift-native = { version = "0.121.0" } +cranelift-jit = { version = "0.121.0", optional = true } +cranelift-object = { version = "0.121.0" } target-lexicon = "0.13" gimli = { version = "0.31", default-features = false, features = ["write"] } object = { version = "0.36", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } @@ -24,12 +24,12 @@ smallvec = "1.8.1" [patch.crates-io] # Uncomment to use an unreleased version of cranelift -#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } -#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } -#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } -#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } -#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } -#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" } +#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" } +#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" } +#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" } +#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" } +#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" } +#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" } # Uncomment to use local checkout of cranelift #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } From e3882618c887f6a6049a99edcf91094a64bd57f3 Mon Sep 17 00:00:00 2001 From: Martin Liska Date: Sun, 22 Jun 2025 16:32:54 +0200 Subject: [PATCH 35/38] cranelift: fix target feature name type: "fxsr" --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 07ea29f3024ef..8e34436fb5e0a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -184,7 +184,7 @@ impl CodegenBackend for CraneliftCodegenBackend { // FIXME return the actually used target features. this is necessary for #[cfg(target_feature)] let target_features = if sess.target.arch == "x86_64" && sess.target.os != "none" { // x86_64 mandates SSE2 support and rustc requires the x87 feature to be enabled - vec![sym::fsxr, sym::sse, sym::sse2, Symbol::intern("x87")] + vec![sym::fxsr, sym::sse, sym::sse2, Symbol::intern("x87")] } else if sess.target.arch == "aarch64" { match &*sess.target.os { "none" => vec![], From 9f5fc01b8b50348091ba3143a4db7dd3e116eda2 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 23 Jun 2025 10:08:13 +0000 Subject: [PATCH 36/38] Rustup to rustc 1.89.0-nightly (be19eda0d 2025-06-22) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index f09a2628777da..a379f9c7e4861 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-19" +channel = "nightly-2025-06-23" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From 8c848e0604b5d26fad120914f822f564fe05c52a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 24 Jun 2025 10:47:30 +0000 Subject: [PATCH 37/38] Rustup to rustc 1.90.0-nightly (706f244db 2025-06-23) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index a379f9c7e4861..150bb562f74a7 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2025-06-23" +channel = "nightly-2025-06-24" components = ["rust-src", "rustc-dev", "llvm-tools"] profile = "minimal" From dbe8682cd1c88dd7027004f7ffdf3d16d27015c4 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 24 Jun 2025 11:38:38 +0000 Subject: [PATCH 38/38] Update tidy exceptions --- src/tools/tidy/src/deps.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 716d42c32ebed..4e0bbcd7c6c20 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -206,6 +206,7 @@ const EXCEPTIONS_CRANELIFT: ExceptionList = &[ ("regalloc2", "Apache-2.0 WITH LLVM-exception"), ("target-lexicon", "Apache-2.0 WITH LLVM-exception"), ("wasmtime-jit-icache-coherence", "Apache-2.0 WITH LLVM-exception"), + ("wasmtime-math", "Apache-2.0 WITH LLVM-exception"), // tidy-alphabetical-end ]; @@ -537,6 +538,7 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ "indexmap", "libc", "libloading", + "libm", "log", "mach2", "memchr", @@ -554,6 +556,7 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ "target-lexicon", "unicode-ident", "wasmtime-jit-icache-coherence", + "wasmtime-math", "windows-sys", "windows-targets", "windows_aarch64_gnullvm",