From db7b096530b252baf840e144b04922747266cdde Mon Sep 17 00:00:00 2001 From: Orson Peters Date: Sun, 6 Jul 2025 01:08:59 +0200 Subject: [PATCH 1/3] Improve TLS codegen by marking the panic/init path as cold --- .../std/src/sys/thread_local/native/eager.rs | 25 ++++++++++++------- .../std/src/sys/thread_local/native/lazy.rs | 3 ++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/library/std/src/sys/thread_local/native/eager.rs b/library/std/src/sys/thread_local/native/eager.rs index fd48c4f720216..c3e016a86825b 100644 --- a/library/std/src/sys/thread_local/native/eager.rs +++ b/library/std/src/sys/thread_local/native/eager.rs @@ -4,7 +4,7 @@ use crate::sys::thread_local::{abort_on_dtor_unwind, destructors}; #[derive(Clone, Copy)] enum State { - Initial, + Uninitialized, Alive, Destroyed, } @@ -17,7 +17,7 @@ pub struct Storage { impl Storage { pub const fn new(val: T) -> Storage { - Storage { state: Cell::new(State::Initial), val: UnsafeCell::new(val) } + Storage { state: Cell::new(State::Uninitialized), val: UnsafeCell::new(val) } } /// Gets a pointer to the TLS value. If the TLS variable has been destroyed, @@ -28,18 +28,25 @@ impl Storage { /// /// # Safety /// The `self` reference must remain valid until the TLS destructor is run. - #[inline] + #[inline(always)] pub unsafe fn get(&self) -> *const T { - match self.state.get() { - State::Alive => self.val.get(), - State::Destroyed => ptr::null(), - State::Initial => unsafe { self.initialize() }, + if let State::Alive = self.state.get() { + self.val.get() + } else { + unsafe { self.get_or_init_slow() } } } #[cold] - unsafe fn initialize(&self) -> *const T { - // Register the destructor + #[inline(never)] + unsafe fn get_or_init_slow(&self) -> *const T { + match self.state.get() { + State::Uninitialized => {} + State::Alive => return self.val.get(), + State::Destroyed => return ptr::null(), + } + + // Register the destructor. // SAFETY: // The caller guarantees that `self` will be valid until thread destruction. diff --git a/library/std/src/sys/thread_local/native/lazy.rs b/library/std/src/sys/thread_local/native/lazy.rs index b556dd9aa25ed..e3ca9c11ffd4f 100644 --- a/library/std/src/sys/thread_local/native/lazy.rs +++ b/library/std/src/sys/thread_local/native/lazy.rs @@ -52,7 +52,7 @@ where /// /// # Safety /// The `self` reference must remain valid until the TLS destructor is run. - #[inline] + #[inline(always)] pub unsafe fn get_or_init(&self, i: Option<&mut Option>, f: impl FnOnce() -> T) -> *const T { if let State::Alive = self.state.get() { self.value.get().cast() @@ -64,6 +64,7 @@ where /// # Safety /// The `self` reference must remain valid until the TLS destructor is run. #[cold] + #[inline(never)] unsafe fn get_or_init_slow( &self, i: Option<&mut Option>, From cf4669ed2a558fc38b8b4ebca8b662aee387785f Mon Sep 17 00:00:00 2001 From: Orson Peters Date: Sun, 6 Jul 2025 01:30:32 +0200 Subject: [PATCH 2/3] Also apply opt to OS-specific TLS impls --- library/std/src/sys/thread_local/key/racy.rs | 4 +++- library/std/src/sys/thread_local/key/windows.rs | 3 ++- library/std/src/sys/thread_local/key/xous.rs | 13 ++++++++++--- library/std/src/sys/thread_local/os.rs | 3 +++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/library/std/src/sys/thread_local/key/racy.rs b/library/std/src/sys/thread_local/key/racy.rs index a12ff7ac36ba5..f0678a28ec9a1 100644 --- a/library/std/src/sys/thread_local/key/racy.rs +++ b/library/std/src/sys/thread_local/key/racy.rs @@ -34,7 +34,7 @@ impl LazyKey { LazyKey { key: AtomicUsize::new(KEY_SENTVAL), dtor } } - #[inline] + #[inline(always)] pub fn force(&self) -> super::Key { match self.key.load(Ordering::Acquire) { KEY_SENTVAL => self.lazy_init() as super::Key, @@ -42,6 +42,8 @@ impl LazyKey { } } + #[cold] + #[inline(never)] fn lazy_init(&self) -> usize { // POSIX allows the key created here to be KEY_SENTVAL, but the compare_exchange // below relies on using KEY_SENTVAL as a sentinel value to check who won the diff --git a/library/std/src/sys/thread_local/key/windows.rs b/library/std/src/sys/thread_local/key/windows.rs index 2ff0fd1196e12..ec085839e4b73 100644 --- a/library/std/src/sys/thread_local/key/windows.rs +++ b/library/std/src/sys/thread_local/key/windows.rs @@ -58,7 +58,7 @@ impl LazyKey { } } - #[inline] + #[inline(always)] pub fn force(&'static self) -> Key { match self.key.load(Acquire) { 0 => unsafe { self.init() }, @@ -67,6 +67,7 @@ impl LazyKey { } #[cold] + #[inline(never)] unsafe fn init(&'static self) -> Key { if self.dtor.is_some() { let mut pending = c::FALSE; diff --git a/library/std/src/sys/thread_local/key/xous.rs b/library/std/src/sys/thread_local/key/xous.rs index a27cec5ca1a60..34dbc71d84a0a 100644 --- a/library/std/src/sys/thread_local/key/xous.rs +++ b/library/std/src/sys/thread_local/key/xous.rs @@ -67,6 +67,7 @@ unsafe extern "Rust" { static DTORS: Atomic<*mut Node>; } +#[inline] fn tls_ptr_addr() -> *mut *mut u8 { let mut tp: usize; unsafe { @@ -80,14 +81,20 @@ fn tls_ptr_addr() -> *mut *mut u8 { /// Creates an area of memory that's unique per thread. This area will /// contain all thread local pointers. +#[inline] fn tls_table() -> &'static mut [*mut u8] { let tp = tls_ptr_addr(); if !tp.is_null() { - return unsafe { - core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / size_of::<*mut u8>()) - }; + unsafe { core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / size_of::<*mut u8>()) } + } else { + tls_table_slow() } +} + +#[cold] +#[inline(never)] +fn tls_table_slow() -> &'static mut [*mut u8] { // If the TP register is `0`, then this thread hasn't initialized // its TLS yet. Allocate a new page to store this memory. let tp = unsafe { diff --git a/library/std/src/sys/thread_local/os.rs b/library/std/src/sys/thread_local/os.rs index fe6af27db3a17..f8fa293127d40 100644 --- a/library/std/src/sys/thread_local/os.rs +++ b/library/std/src/sys/thread_local/os.rs @@ -68,6 +68,7 @@ impl Storage { /// /// The resulting pointer may not be used after reentrant inialialization /// or thread destruction has occurred. + #[inline] pub fn get(&'static self, i: Option<&mut Option>, f: impl FnOnce() -> T) -> *const T { let key = self.key.force(); let ptr = unsafe { get(key) as *mut Value }; @@ -84,6 +85,8 @@ impl Storage { /// # Safety /// * `key` must be the result of calling `self.key.force()` /// * `ptr` must be the current value associated with `key`. + #[cold] + #[inline(never)] unsafe fn try_initialize( key: Key, ptr: *mut Value, From 92fa8e8dafbd8ce62f0d1381e2a0816c5aa58f85 Mon Sep 17 00:00:00 2001 From: Orson Peters Date: Sun, 6 Jul 2025 15:36:53 +0200 Subject: [PATCH 3/3] Don't use inline(never) --- library/std/src/sys/thread_local/key/racy.rs | 3 +-- library/std/src/sys/thread_local/key/windows.rs | 3 +-- library/std/src/sys/thread_local/key/xous.rs | 1 - library/std/src/sys/thread_local/native/eager.rs | 3 +-- library/std/src/sys/thread_local/native/lazy.rs | 3 +-- library/std/src/sys/thread_local/os.rs | 1 - 6 files changed, 4 insertions(+), 10 deletions(-) diff --git a/library/std/src/sys/thread_local/key/racy.rs b/library/std/src/sys/thread_local/key/racy.rs index f0678a28ec9a1..0a10925ae6499 100644 --- a/library/std/src/sys/thread_local/key/racy.rs +++ b/library/std/src/sys/thread_local/key/racy.rs @@ -34,7 +34,7 @@ impl LazyKey { LazyKey { key: AtomicUsize::new(KEY_SENTVAL), dtor } } - #[inline(always)] + #[inline] pub fn force(&self) -> super::Key { match self.key.load(Ordering::Acquire) { KEY_SENTVAL => self.lazy_init() as super::Key, @@ -43,7 +43,6 @@ impl LazyKey { } #[cold] - #[inline(never)] fn lazy_init(&self) -> usize { // POSIX allows the key created here to be KEY_SENTVAL, but the compare_exchange // below relies on using KEY_SENTVAL as a sentinel value to check who won the diff --git a/library/std/src/sys/thread_local/key/windows.rs b/library/std/src/sys/thread_local/key/windows.rs index ec085839e4b73..2ff0fd1196e12 100644 --- a/library/std/src/sys/thread_local/key/windows.rs +++ b/library/std/src/sys/thread_local/key/windows.rs @@ -58,7 +58,7 @@ impl LazyKey { } } - #[inline(always)] + #[inline] pub fn force(&'static self) -> Key { match self.key.load(Acquire) { 0 => unsafe { self.init() }, @@ -67,7 +67,6 @@ impl LazyKey { } #[cold] - #[inline(never)] unsafe fn init(&'static self) -> Key { if self.dtor.is_some() { let mut pending = c::FALSE; diff --git a/library/std/src/sys/thread_local/key/xous.rs b/library/std/src/sys/thread_local/key/xous.rs index 34dbc71d84a0a..f647bf59ff9e1 100644 --- a/library/std/src/sys/thread_local/key/xous.rs +++ b/library/std/src/sys/thread_local/key/xous.rs @@ -93,7 +93,6 @@ fn tls_table() -> &'static mut [*mut u8] { } #[cold] -#[inline(never)] fn tls_table_slow() -> &'static mut [*mut u8] { // If the TP register is `0`, then this thread hasn't initialized // its TLS yet. Allocate a new page to store this memory. diff --git a/library/std/src/sys/thread_local/native/eager.rs b/library/std/src/sys/thread_local/native/eager.rs index c3e016a86825b..b91c072b5d892 100644 --- a/library/std/src/sys/thread_local/native/eager.rs +++ b/library/std/src/sys/thread_local/native/eager.rs @@ -28,7 +28,7 @@ impl Storage { /// /// # Safety /// The `self` reference must remain valid until the TLS destructor is run. - #[inline(always)] + #[inline] pub unsafe fn get(&self) -> *const T { if let State::Alive = self.state.get() { self.val.get() @@ -38,7 +38,6 @@ impl Storage { } #[cold] - #[inline(never)] unsafe fn get_or_init_slow(&self) -> *const T { match self.state.get() { State::Uninitialized => {} diff --git a/library/std/src/sys/thread_local/native/lazy.rs b/library/std/src/sys/thread_local/native/lazy.rs index e3ca9c11ffd4f..b556dd9aa25ed 100644 --- a/library/std/src/sys/thread_local/native/lazy.rs +++ b/library/std/src/sys/thread_local/native/lazy.rs @@ -52,7 +52,7 @@ where /// /// # Safety /// The `self` reference must remain valid until the TLS destructor is run. - #[inline(always)] + #[inline] pub unsafe fn get_or_init(&self, i: Option<&mut Option>, f: impl FnOnce() -> T) -> *const T { if let State::Alive = self.state.get() { self.value.get().cast() @@ -64,7 +64,6 @@ where /// # Safety /// The `self` reference must remain valid until the TLS destructor is run. #[cold] - #[inline(never)] unsafe fn get_or_init_slow( &self, i: Option<&mut Option>, diff --git a/library/std/src/sys/thread_local/os.rs b/library/std/src/sys/thread_local/os.rs index f8fa293127d40..a0c6d16a5d020 100644 --- a/library/std/src/sys/thread_local/os.rs +++ b/library/std/src/sys/thread_local/os.rs @@ -86,7 +86,6 @@ impl Storage { /// * `key` must be the result of calling `self.key.force()` /// * `ptr` must be the current value associated with `key`. #[cold] - #[inline(never)] unsafe fn try_initialize( key: Key, ptr: *mut Value,