Skip to content

Commit 4e0d0d7

Browse files
borsbors
committed
Auto merge of #102750 - the8472:opt-field-order, r=wesleywiser
optimize field ordering by grouping m*2^n-sized fields with equivalently aligned ones ```rust use std::ptr::addr_of; use std::mem; struct Foo { word: u32, byte: u8, ary: [u8; 4] } fn main() { let foo: Foo = unsafe { mem::zeroed() }; println!("base: {:p}\nword: {:p}\nbyte: {:p}\nary: {:p}", &foo, addr_of!(foo.word), addr_of!(foo.byte), addr_of!(foo.ary)); } ``` prints ``` base: 0x7fffc1a8a668 word: 0x7fffc1a8a668 byte: 0x7fffc1a8a66c ary: 0x7fffc1a8a66d ``` I.e. the `u8` in the middle causes the array to sit at an odd offset, which might prevent optimizations, especially on architectures where unaligned loads are costly. Note that this will make field ordering niche-dependent, i.e. a `Bar<T>` with `T=char` and `T=u32` may result in different field order, this may break some code that makes invalid assumptions about `repr(Rust)` types.
2 parents 3f2b2ee + c1f392d commit 4e0d0d7

File tree

9 files changed

+78
-31
lines changed

9 files changed

+78
-31
lines changed

‎compiler/rustc_hir/src/hir.rs

+8-1
Original file line numberDiff line numberDiff line change
@@ -3594,9 +3594,16 @@ mod size_asserts {
35943594
static_assert_size!(Res, 12);
35953595
static_assert_size!(Stmt<'_>, 32);
35963596
static_assert_size!(StmtKind<'_>, 16);
3597+
// tidy-alphabetical-end
3598+
// FIXME: move the tidy directive to the end after the next bootstrap bump
3599+
#[cfg(bootstrap)]
35973600
static_assert_size!(TraitItem<'_>, 88);
3601+
#[cfg(not(bootstrap))]
3602+
static_assert_size!(TraitItem<'_>, 80);
3603+
#[cfg(bootstrap)]
35983604
static_assert_size!(TraitItemKind<'_>, 48);
3605+
#[cfg(not(bootstrap))]
3606+
static_assert_size!(TraitItemKind<'_>, 40);
35993607
static_assert_size!(Ty<'_>, 48);
36003608
static_assert_size!(TyKind<'_>, 32);
3601-
// tidy-alphabetical-end
36023609
}

‎compiler/rustc_ty_utils/src/layout.rs

+23-5
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,18 @@ fn univariant_uninterned<'tcx>(
138138
if optimize {
139139
let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
140140
let optimizing = &mut inverse_memory_index[..end];
141-
let field_align = |f: &TyAndLayout<'_>| {
142-
if let Some(pack) = pack { f.align.abi.min(pack) } else { f.align.abi }
141+
let effective_field_align = |f: &TyAndLayout<'_>| {
142+
if let Some(pack) = pack {
143+
// return the packed alignment in bytes
144+
f.align.abi.min(pack).bytes()
145+
} else {
146+
// returns log2(effective-align).
147+
// This is ok since `pack` applies to all fields equally.
148+
// The calculation assumes that size is an integer multiple of align, except for ZSTs.
149+
//
150+
// group [u8; 4] with align-4 or [u8; 6] with align-2 fields
151+
f.align.abi.bytes().max(f.size.bytes()).trailing_zeros() as u64
152+
}
143153
};
144154

145155
// If `-Z randomize-layout` was enabled for the type definition we can shuffle
@@ -160,15 +170,23 @@ fn univariant_uninterned<'tcx>(
160170
optimizing.sort_by_key(|&x| {
161171
// Place ZSTs first to avoid "interesting offsets",
162172
// especially with only one or two non-ZST fields.
173+
// Then place largest alignments first, largest niches within an alignment group last
163174
let f = &fields[x as usize];
164-
(!f.is_zst(), cmp::Reverse(field_align(f)))
175+
let niche_size = f.largest_niche.map_or(0, |n| n.available(cx));
176+
(!f.is_zst(), cmp::Reverse(effective_field_align(f)), niche_size)
165177
});
166178
}
167179

168180
StructKind::Prefixed(..) => {
169181
// Sort in ascending alignment so that the layout stays optimal
170-
// regardless of the prefix
171-
optimizing.sort_by_key(|&x| field_align(&fields[x as usize]));
182+
// regardless of the prefix.
183+
// And put the largest niche in an alignment group at the end
184+
// so it can be used as discriminant in jagged enums
185+
optimizing.sort_by_key(|&x| {
186+
let f = &fields[x as usize];
187+
let niche_size = f.largest_niche.map_or(0, |n| n.available(cx));
188+
(effective_field_align(f), niche_size)
189+
});
172190
}
173191
}
174192

‎src/test/codegen/issue-37945.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use std::slice::Iter;
1515
pub fn is_empty_1(xs: Iter<f32>) -> bool {
1616
// CHECK-LABEL: @is_empty_1(
1717
// CHECK-NEXT: start:
18-
// CHECK-NEXT: [[A:%.*]] = icmp ne {{i32\*|ptr}} %xs.1, null
18+
// CHECK-NEXT: [[A:%.*]] = icmp ne {{i32\*|ptr}} {{%xs.0|%xs.1}}, null
1919
// CHECK-NEXT: tail call void @llvm.assume(i1 [[A]])
2020
// The order between %xs.0 and %xs.1 on the next line doesn't matter
2121
// and different LLVM versions produce different order.
@@ -28,7 +28,7 @@ pub fn is_empty_1(xs: Iter<f32>) -> bool {
2828
pub fn is_empty_2(xs: Iter<f32>) -> bool {
2929
// CHECK-LABEL: @is_empty_2
3030
// CHECK-NEXT: start:
31-
// CHECK-NEXT: [[C:%.*]] = icmp ne {{i32\*|ptr}} %xs.1, null
31+
// CHECK-NEXT: [[C:%.*]] = icmp ne {{i32\*|ptr}} {{%xs.0|%xs.1}}, null
3232
// CHECK-NEXT: tail call void @llvm.assume(i1 [[C]])
3333
// The order between %xs.0 and %xs.1 on the next line doesn't matter
3434
// and different LLVM versions produce different order.

‎src/test/codegen/mem-replace-direct-memcpy.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ pub fn replace_byte(dst: &mut u8, src: u8) -> u8 {
1818
// CHECK-NOT: call void @llvm.memcpy
1919
// CHECK: ; core::mem::replace
2020
// CHECK-NOT: call void @llvm.memcpy
21-
// CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 1 %{{.*}}, {{i8\*|ptr}} align 1 %dest, i{{.*}} 1, i1 false)
21+
// CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 1 %{{.*}}, {{i8\*|ptr}} align 1 %{{.*}}, i{{.*}} 1, i1 false)
2222
// CHECK-NOT: call void @llvm.memcpy
23-
// CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 1 %dest, {{i8\*|ptr}} align 1 %src{{.*}}, i{{.*}} 1, i1 false)
23+
// CHECK: call void @llvm.memcpy.{{.+}}({{i8\*|ptr}} align 1 %{{.*}}, {{i8\*|ptr}} align 1 %{{.*}}, i{{.*}} 1, i1 false)
2424
// CHECK-NOT: call void @llvm.memcpy

‎src/test/codegen/slice-iter-len-eq-zero.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ type Demo = [u8; 3];
99
#[no_mangle]
1010
pub fn slice_iter_len_eq_zero(y: std::slice::Iter<'_, Demo>) -> bool {
1111
// CHECK-NOT: sub
12-
// CHECK: %2 = icmp eq {{i8\*|ptr}} %1, %0
12+
// CHECK: %2 = icmp eq {{i8\*|ptr}} {{%1|%0}}, {{%1|%0}}
1313
// CHECK: ret i1 %2
1414
y.len() == 0
1515
}

‎src/test/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr

+10-10
Original file line numberDiff line numberDiff line change
@@ -370,31 +370,31 @@ error: layout_of(NicheFirst) = Layout {
370370
pref: $PREF_ALIGN,
371371
},
372372
abi: ScalarPair(
373-
Initialized {
373+
Union {
374374
value: Int(
375375
I8,
376376
false,
377377
),
378-
valid_range: 0..=4,
379378
},
380-
Union {
379+
Initialized {
381380
value: Int(
382381
I8,
383382
false,
384383
),
384+
valid_range: 0..=4,
385385
},
386386
),
387387
fields: Arbitrary {
388388
offsets: [
389-
Size(0 bytes),
389+
Size(1 bytes),
390390
],
391391
memory_index: [
392392
0,
393393
],
394394
},
395395
largest_niche: Some(
396396
Niche {
397-
offset: Size(0 bytes),
397+
offset: Size(1 bytes),
398398
value: Int(
399399
I8,
400400
false,
@@ -429,29 +429,29 @@ error: layout_of(NicheFirst) = Layout {
429429
I8,
430430
false,
431431
),
432-
valid_range: 0..=2,
432+
valid_range: 0..=255,
433433
},
434434
Initialized {
435435
value: Int(
436436
I8,
437437
false,
438438
),
439-
valid_range: 0..=255,
439+
valid_range: 0..=2,
440440
},
441441
),
442442
fields: Arbitrary {
443443
offsets: [
444-
Size(0 bytes),
445444
Size(1 bytes),
445+
Size(0 bytes),
446446
],
447447
memory_index: [
448-
0,
449448
1,
449+
0,
450450
],
451451
},
452452
largest_niche: Some(
453453
Niche {
454-
offset: Size(0 bytes),
454+
offset: Size(1 bytes),
455455
value: Int(
456456
I8,
457457
false,

‎src/test/ui/stats/hir-stats.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// check-pass
22
// compile-flags: -Zhir-stats
33
// only-x86_64
4+
// ignore-stage1 FIXME: remove after next bootstrap bump
45

56
// The aim here is to include at least one of every different type of top-level
67
// AST/HIR node reported by `-Zhir-stats`.

‎src/test/ui/stats/hir-stats.stderr

+10-10
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@ ast-stats-1 PRE EXPANSION AST STATS
22
ast-stats-1 Name Accumulated Size Count Item Size
33
ast-stats-1 ----------------------------------------------------------------
44
ast-stats-1 ExprField 48 ( 0.6%) 1 48
5+
ast-stats-1 GenericArgs 56 ( 0.8%) 1 56
6+
ast-stats-1 - AngleBracketed 56 ( 0.8%) 1
57
ast-stats-1 Crate 56 ( 0.8%) 1 56
68
ast-stats-1 Attribute 64 ( 0.9%) 2 32
79
ast-stats-1 - Normal 32 ( 0.4%) 1
810
ast-stats-1 - DocComment 32 ( 0.4%) 1
9-
ast-stats-1 GenericArgs 64 ( 0.9%) 1 64
10-
ast-stats-1 - AngleBracketed 64 ( 0.9%) 1
1111
ast-stats-1 Local 72 ( 1.0%) 1 72
1212
ast-stats-1 WherePredicate 72 ( 1.0%) 1 72
1313
ast-stats-1 - BoundPredicate 72 ( 1.0%) 1
@@ -53,15 +53,15 @@ ast-stats-1 - Impl 184 ( 2.5%) 1
5353
ast-stats-1 - Fn 368 ( 5.0%) 2
5454
ast-stats-1 - Use 552 ( 7.4%) 3
5555
ast-stats-1 ----------------------------------------------------------------
56-
ast-stats-1 Total 7_424
56+
ast-stats-1 Total 7_416
5757
ast-stats-1
5858
ast-stats-2 POST EXPANSION AST STATS
5959
ast-stats-2 Name Accumulated Size Count Item Size
6060
ast-stats-2 ----------------------------------------------------------------
6161
ast-stats-2 ExprField 48 ( 0.6%) 1 48
62+
ast-stats-2 GenericArgs 56 ( 0.7%) 1 56
63+
ast-stats-2 - AngleBracketed 56 ( 0.7%) 1
6264
ast-stats-2 Crate 56 ( 0.7%) 1 56
63-
ast-stats-2 GenericArgs 64 ( 0.8%) 1 64
64-
ast-stats-2 - AngleBracketed 64 ( 0.8%) 1
6565
ast-stats-2 Local 72 ( 0.9%) 1 72
6666
ast-stats-2 WherePredicate 72 ( 0.9%) 1 72
6767
ast-stats-2 - BoundPredicate 72 ( 0.9%) 1
@@ -80,9 +80,9 @@ ast-stats-2 - Expr 96 ( 1.2%) 3
8080
ast-stats-2 Param 160 ( 2.0%) 4 40
8181
ast-stats-2 FnDecl 200 ( 2.5%) 5 40
8282
ast-stats-2 Variant 240 ( 3.0%) 2 120
83-
ast-stats-2 GenericBound 288 ( 3.5%) 4 72
84-
ast-stats-2 - Trait 288 ( 3.5%) 4
85-
ast-stats-2 Block 288 ( 3.5%) 6 48
83+
ast-stats-2 GenericBound 288 ( 3.6%) 4 72
84+
ast-stats-2 - Trait 288 ( 3.6%) 4
85+
ast-stats-2 Block 288 ( 3.6%) 6 48
8686
ast-stats-2 AssocItem 416 ( 5.1%) 4 104
8787
ast-stats-2 - Type 208 ( 2.6%) 2
8888
ast-stats-2 - Fn 208 ( 2.6%) 2
@@ -104,7 +104,7 @@ ast-stats-2 - Rptr 64 ( 0.8%) 1
104104
ast-stats-2 - Ptr 64 ( 0.8%) 1
105105
ast-stats-2 - ImplicitSelf 128 ( 1.6%) 2
106106
ast-stats-2 - Path 640 ( 7.9%) 10
107-
ast-stats-2 Item 2_024 (24.9%) 11 184
107+
ast-stats-2 Item 2_024 (25.0%) 11 184
108108
ast-stats-2 - Trait 184 ( 2.3%) 1
109109
ast-stats-2 - Enum 184 ( 2.3%) 1
110110
ast-stats-2 - ExternCrate 184 ( 2.3%) 1
@@ -113,7 +113,7 @@ ast-stats-2 - Impl 184 ( 2.3%) 1
113113
ast-stats-2 - Fn 368 ( 4.5%) 2
114114
ast-stats-2 - Use 736 ( 9.1%) 4
115115
ast-stats-2 ----------------------------------------------------------------
116-
ast-stats-2 Total 8_120
116+
ast-stats-2 Total 8_112
117117
ast-stats-2
118118
hir-stats HIR STATS
119119
hir-stats Name Accumulated Size Count Item Size

‎src/test/ui/structs-enums/type-sizes.rs

+21
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#![allow(non_camel_case_types)]
44
#![allow(dead_code)]
55
#![feature(never_type)]
6+
#![feature(pointer_is_aligned)]
67

78
use std::mem::size_of;
89
use std::num::NonZeroU8;
@@ -168,6 +169,18 @@ pub enum EnumManyVariant<X> {
168169
_F0, _F1, _F2, _F3, _F4, _F5, _F6, _F7, _F8, _F9, _FA, _FB, _FC, _FD, _FE, _FF,
169170
}
170171

172+
struct Reorder4 {
173+
a: u32,
174+
b: u8,
175+
ary: [u8; 4],
176+
}
177+
178+
struct Reorder2 {
179+
a: u16,
180+
b: u8,
181+
ary: [u8; 6],
182+
}
183+
171184
pub fn main() {
172185
assert_eq!(size_of::<u8>(), 1 as usize);
173186
assert_eq!(size_of::<u32>(), 4 as usize);
@@ -249,4 +262,12 @@ pub fn main() {
249262
assert_eq!(size_of::<EnumManyVariant<Option<NicheU16>>>(), 4);
250263
assert_eq!(size_of::<EnumManyVariant<Option2<NicheU16,u8>>>(), 6);
251264
assert_eq!(size_of::<EnumManyVariant<Option<(NicheU16,u8)>>>(), 6);
265+
266+
267+
let v = Reorder4 {a: 0, b: 0, ary: [0; 4]};
268+
assert_eq!(size_of::<Reorder4>(), 12);
269+
assert!((&v.ary).as_ptr().is_aligned_to(4), "[u8; 4] should group with align-4 fields");
270+
let v = Reorder2 {a: 0, b: 0, ary: [0; 6]};
271+
assert_eq!(size_of::<Reorder2>(), 10);
272+
assert!((&v.ary).as_ptr().is_aligned_to(2), "[u8; 6] should group with align-2 fields");
252273
}

0 commit comments

Comments
 (0)