Auto merge of #143511 - orlp:tls-cold-init, r=<try>

Improve TLS codegen by marking the panic/init path as cold

This is an extension of the performance improvements seen from <#141685>. I noticed that the non-`const` TLS still didn't have the `#[cold]` attribute for the uninit/panic path, and I also realized that neither implementation should have the initialization or panic path inlined, ever.

These paths are taken either only once per thread (`init`) or never (`panic`, in a well-behaving Rust program), thus they don't deserve to litter the code generated each time you access a thread-local variable. So in addition to `#[cold]` I added the more aggressive `#[inline(never)]` to both cold paths as well.

Author: rust-bors[bot]

library/std/src/sys/thread_local/key/racy.rs

@@ -42,6 +42,7 @@ impl LazyKey {
         }
     }
 
+    #[cold]
     fn lazy_init(&self) -> usize {
         // POSIX allows the key created here to be KEY_SENTVAL, but the compare_exchange
         // below relies on using KEY_SENTVAL as a sentinel value to check who won the

library/std/src/sys/thread_local/key/xous.rs

@@ -67,6 +67,7 @@ unsafe extern "Rust" {
     static DTORS: Atomic<*mut Node>;
 }
 
+#[inline]
 fn tls_ptr_addr() -> *mut *mut u8 {
     let mut tp: usize;
     unsafe {
@@ -80,14 +81,19 @@ fn tls_ptr_addr() -> *mut *mut u8 {
 
 /// Creates an area of memory that's unique per thread. This area will
 /// contain all thread local pointers.
+#[inline]
 fn tls_table() -> &'static mut [*mut u8] {
     let tp = tls_ptr_addr();
 
     if !tp.is_null() {
-        return unsafe {
-            core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / size_of::<*mut u8>())
-        };
+        unsafe { core::slice::from_raw_parts_mut(tp, TLS_MEMORY_SIZE / size_of::<*mut u8>()) }
+    } else {
+        tls_table_slow()
     }
+}
+
+#[cold]
+fn tls_table_slow() -> &'static mut [*mut u8] {
     // If the TP register is `0`, then this thread hasn't initialized
     // its TLS yet. Allocate a new page to store this memory.
     let tp = unsafe {

library/std/src/sys/thread_local/native/eager.rs

@@ -4,7 +4,7 @@ use crate::sys::thread_local::{abort_on_dtor_unwind, destructors};
 
 #[derive(Clone, Copy)]
 enum State {
-    Initial,
+    Uninitialized,
     Alive,
     Destroyed,
 }
@@ -17,7 +17,7 @@ pub struct Storage<T> {
 
 impl<T> Storage<T> {
     pub const fn new(val: T) -> Storage<T> {
-        Storage { state: Cell::new(State::Initial), val: UnsafeCell::new(val) }
+        Storage { state: Cell::new(State::Uninitialized), val: UnsafeCell::new(val) }
     }
 
     /// Gets a pointer to the TLS value. If the TLS variable has been destroyed,
@@ -30,16 +30,22 @@ impl<T> Storage<T> {
     /// The `self` reference must remain valid until the TLS destructor is run.
     #[inline]
     pub unsafe fn get(&self) -> *const T {
-        match self.state.get() {
-            State::Alive => self.val.get(),
-            State::Destroyed => ptr::null(),
-            State::Initial => unsafe { self.initialize() },
+        if let State::Alive = self.state.get() {
+            self.val.get()
+        } else {
+            unsafe { self.get_or_init_slow() }
         }
     }
 
     #[cold]
-    unsafe fn initialize(&self) -> *const T {
-        // Register the destructor
+    unsafe fn get_or_init_slow(&self) -> *const T {
+        match self.state.get() {
+            State::Uninitialized => {}
+            State::Alive => return self.val.get(),
+            State::Destroyed => return ptr::null(),
+        }
+
+        // Register the destructor.
 
         // SAFETY:
         // The caller guarantees that `self` will be valid until thread destruction.

library/std/src/sys/thread_local/os.rs

@@ -68,6 +68,7 @@ impl<T: 'static> Storage<T> {
     ///
     /// The resulting pointer may not be used after reentrant inialialization
     /// or thread destruction has occurred.
+    #[inline]
     pub fn get(&'static self, i: Option<&mut Option<T>>, f: impl FnOnce() -> T) -> *const T {
         let key = self.key.force();
         let ptr = unsafe { get(key) as *mut Value<T> };
@@ -84,6 +85,7 @@ impl<T: 'static> Storage<T> {
     /// # Safety
     /// * `key` must be the result of calling `self.key.force()`
     /// * `ptr` must be the current value associated with `key`.
+    #[cold]
     unsafe fn try_initialize(
         key: Key,
         ptr: *mut Value<T>,