exclude unsizable tail from randomization seed calculation

Author: the8472

compiler/rustc_abi/src/layout.rs

@@ -1051,15 +1051,16 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
         let mut max_repr_align = repr.align;
         let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
-        let field_seed =
-            fields.raw.iter().fold(0u64, |acc, f| acc.wrapping_add(f.randomization_seed));
         let optimize_field_order = !repr.inhibit_struct_field_reordering();
-        if optimize_field_order && fields.len() > 1 {
-            let end =
-                if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
-            let optimizing = &mut inverse_memory_index.raw[..end];
-            let fields_excluding_tail = &fields.raw[..end];
+        let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
+        let optimizing = &mut inverse_memory_index.raw[..end];
+        let fields_excluding_tail = &fields.raw[..end];
+        // unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts.
+        let field_seed = fields_excluding_tail
+            .iter()
+            .fold(0u64, |acc, f| acc.wrapping_add(f.randomization_seed));
 
+        if optimize_field_order && fields.len() > 1 {
             // If `-Z randomize-layout` was enabled for the type definition we can shuffle
             // the field ordering to try and catch some code making assumptions about layouts
             // we don't guarantee.
@@ -1369,12 +1370,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             unadjusted_abi_align
         };
 
-        // a transparent struct only has a single field, so its seed should be the same as the one we pass forward
-        let seed = if repr.transparent() {
-            field_seed
-        } else {
-            field_seed.wrapping_add(repr.field_shuffle_seed)
-        };
+        let seed = field_seed.wrapping_add(repr.field_shuffle_seed);
 
         Ok(LayoutData {
             variants: Variants::Single { index: VariantIdx::new(0) },

tests/ui/layout/randomize.rs

@@ -20,10 +20,11 @@ const _: () = {
     #[cfg(randomize_layout)]
     assert!(std::mem::offset_of!(Foo::<u16>, 1) != std::mem::offset_of!(Foo::<Wrapper<u16>>, 1));
 
-    // but repr(transparent) should make them the same again.
-    // maybe not strictly guaranteed? but UCG has been leaning in that direction at least
+    // Even transparent wrapper inner types get a different layout since associated type
+    // pecialization could result in the outer type behaving differently depending on the exact
+    // inner type.
     #[cfg(randomize_layout)]
     assert!(
-        std::mem::offset_of!(Foo::<u16>, 1) == std::mem::offset_of!(Foo::<TransparentWrapper>, 1)
+        std::mem::offset_of!(Foo::<u16>, 1) != std::mem::offset_of!(Foo::<TransparentWrapper>, 1)
     );
 };