diff --git a/src/librustc/ty/layout.rs b/src/librustc/ty/layout.rs index 8febcfd0754c9..dd7f93fb6cf91 100644 --- a/src/librustc/ty/layout.rs +++ b/src/librustc/ty/layout.rs @@ -267,33 +267,61 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { } } + fn align_and_pack(&self, repr: &ReprOptions) -> (AbiAndPrefAlign, Option) { + let dl = self.data_layout(); + if repr.packed() { + let pack = Align::from_bytes(repr.pack as u64).unwrap(); + if repr.align > 0 { + bug!("adt cannot be packed and aligned"); + } + (dl.i8_align, Some(pack)) + } else { + let align = Align::from_bytes(repr.align as u64).unwrap(); + (dl.aggregate_align.max(AbiAndPrefAlign::new(align)), None) + } + } + + fn variant_size(&self, + fields: &[TyLayout<'_>], + repr: &ReprOptions) -> Option<(Size, AbiAndPrefAlign)> { + let dl = self.data_layout(); + let (mut align, pack) = self.align_and_pack(repr); + + let optimize = !repr.inhibit_struct_field_reordering_opt(); + + let mut size = Size::ZERO; + for field in fields.iter() { + assert!(!field.is_unsized()); + let field_align = if let Some(pack) = pack { + field.align.min(AbiAndPrefAlign::new(pack)) + } else { + field.align + }; + if !optimize { + size = size.align_to(field_align.abi); + } + align = align.max(field_align); + size = size.checked_add(field.size, dl)?; + } + if !optimize { + size = size.align_to(align.abi); + } + Some((size, align)) + } + fn univariant_uninterned(&self, ty: Ty<'tcx>, fields: &[TyLayout<'_>], repr: &ReprOptions, kind: StructKind) -> Result> { let dl = self.data_layout(); - let packed = repr.packed(); - if packed && repr.align > 0 { - bug!("struct cannot be packed and aligned"); - } - - let pack = Align::from_bytes(repr.pack as u64).unwrap(); - - let mut align = if packed { - dl.i8_align - } else { - dl.aggregate_align - }; + let (mut align, pack) = self.align_and_pack(repr); let mut sized = true; let mut offsets = vec![Size::ZERO; fields.len()]; let mut inverse_memory_index: Vec = (0..fields.len() as u32).collect(); - let mut optimize = !repr.inhibit_struct_field_reordering_opt(); - if let StructKind::Prefixed(_, align) = kind { - optimize &= align.bytes() == 1; - } + let optimize = !repr.inhibit_struct_field_reordering_opt(); if optimize { let end = if let StructKind::MaybeUnsized = kind { @@ -303,7 +331,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { }; let optimizing = &mut inverse_memory_index[..end]; let field_align = |f: &TyLayout<'_>| { - if packed { f.align.abi.min(pack) } else { f.align.abi } + if let Some(pack) = pack { f.align.abi.min(pack) } else { f.align.abi } }; match kind { StructKind::AlwaysSized | @@ -334,7 +362,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { let mut largest_niche_available = 0; if let StructKind::Prefixed(prefix_size, prefix_align) = kind { - let prefix_align = if packed { + let prefix_align = if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align @@ -355,7 +383,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { } // Invariant: offset < dl.obj_size_bound() <= 1<<61 - let field_align = if packed { + let field_align = if let Some(pack) = pack { field.align.min(AbiAndPrefAlign::new(pack)) } else { field.align @@ -379,12 +407,6 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { .ok_or(LayoutError::SizeOverflow(ty))?; } - if repr.align > 0 { - let repr_align = repr.align as u64; - align = align.max(AbiAndPrefAlign::new(Align::from_bytes(repr_align).unwrap())); - debug!("univariant repr_align: {:?}", repr_align); - } - debug!("univariant min_size: {:?}", offset); let min_size = offset; @@ -730,24 +752,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { }).collect::, _>>()?; if def.is_union() { - let packed = def.repr.packed(); - if packed && def.repr.align > 0 { - bug!("Union cannot be packed and aligned"); - } - - let pack = Align::from_bytes(def.repr.pack as u64).unwrap(); - - let mut align = if packed { - dl.i8_align - } else { - dl.aggregate_align - }; - - if def.repr.align > 0 { - let repr_align = def.repr.align as u64; - align = align.max( - AbiAndPrefAlign::new(Align::from_bytes(repr_align).unwrap())); - } + let (mut align, pack) = self.align_and_pack(&def.repr); let optimize = !def.repr.inhibit_union_abi_opt(); let mut size = Size::ZERO; @@ -756,7 +761,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { for field in &variants[index] { assert!(!field.is_unsized()); - let field_align = if packed { + let field_align = if let Some(pack) = pack { field.align.min(AbiAndPrefAlign::new(pack)) } else { field.align @@ -906,6 +911,74 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { return Ok(tcx.intern_layout(st)); } + let mut align = dl.i8_align; + let mut max_size = Size::ZERO; + + for fields in variants.iter() { + let (v_size, v_align) = self.variant_size(fields, &def.repr) + .ok_or(LayoutError::SizeOverflow(ty))?; + max_size = max_size.max(v_size); + align = align.max(v_align); + } + + let mut gap = max_size.align_to(align.abi) - max_size; + + let (mut min, mut max) = (i128::max_value(), i128::min_value()); + let discr_type = def.repr.discr_type(); + let bits = Integer::from_attr(self, discr_type).size().bits(); + for (i, discr) in def.discriminants(tcx) { + if variants[i].iter().any(|f| f.abi.is_uninhabited()) { + continue; + } + let mut x = discr.val as i128; + if discr_type.is_signed() { + // sign extend the raw representation to be an i128 + x = (x << (128 - bits)) >> (128 - bits); + } + if x < min { min = x; } + if x > max { max = x; } + } + // We might have no inhabited variants, so pretend there's at least one. + if (min, max) == (i128::max_value(), i128::min_value()) { + min = 0; + max = 0; + } + assert!(min <= max, "discriminant range is {}...{}", min, max); + let (min_ity, signed) = Integer::repr_discr(tcx, ty, &def.repr, min, max); + + let typeck_ity = Integer::from_attr(dl, def.repr.discr_type()); + if typeck_ity < min_ity { + // It is a bug if Layout decided on a greater discriminant size than typeck for + // some reason at this point (based on values discriminant can take on). Mostly + // because this discriminant will be loaded, and then stored into variable of + // type calculated by typeck. Consider such case (a bug): typeck decided on + // byte-sized discriminant, but layout thinks we need a 16-bit to store all + // discriminant values. That would be a bug, because then, in codegen, in order + // to store this 16-bit discriminant into 8-bit sized temporary some of the + // space necessary to represent would have to be discarded (or layout is wrong + // on thinking it needs 16 bits) + bug!("layout decided on a larger discriminant type ({:?}) than typeck ({:?})", + min_ity, typeck_ity); + // However, it is fine to make discr type however large (as an optimisation) + // after this point – we’ll just truncate the value we load in codegen. + } + + let must_grow = min_ity.size() > gap; + if must_grow { + gap += (min_ity.size() - gap).align_to(align.abi); + } + + // We increase the size of the discriminant to avoid LLVM copying + // padding when it doesn't need to. This normally causes unaligned + // load/stores and excessive memcpy/memset operations. By using a + // bigger integer size, LLVM can be sure about its contents and + // won't be so conservative. + let ity = if def.repr.inhibit_enum_layout_opt() { + min_ity + } else { + Integer::approximate_size(gap).unwrap() + }; + // The current code for niche-filling relies on variant indices // instead of actual discriminants, so dataful enums with // explicit discriminants (RFC #2363) would misbehave. @@ -951,19 +1024,20 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { Some(niche) => niche, _ => continue, }; + // Do not use a niche smaller than the tag unless it reduces the size. + if !must_grow && niche.available(dl) >> ity.size().bits() == 0 { + continue; + } let (niche_start, niche_scalar) = match niche.reserve(self, count) { Some(pair) => pair, None => continue, }; - let mut align = dl.aggregate_align; let st = variants.iter_enumerated().map(|(j, v)| { let mut st = self.univariant_uninterned(ty, v, &def.repr, StructKind::AlwaysSized)?; st.variants = Variants::Single { index: j }; - - align = align.max(st.align); - + assert!(st.align.abi <= align.abi && st.align.pref <= align.pref); Ok(st) }).collect::, _>>()?; @@ -1025,65 +1099,27 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { } } - let (mut min, mut max) = (i128::max_value(), i128::min_value()); - let discr_type = def.repr.discr_type(); - let bits = Integer::from_attr(self, discr_type).size().bits(); - for (i, discr) in def.discriminants(tcx) { - if variants[i].iter().any(|f| f.abi.is_uninhabited()) { - continue; - } - let mut x = discr.val as i128; - if discr_type.is_signed() { - // sign extend the raw representation to be an i128 - x = (x << (128 - bits)) >> (128 - bits); - } - if x < min { min = x; } - if x > max { max = x; } - } - // We might have no inhabited variants, so pretend there's at least one. - if (min, max) == (i128::max_value(), i128::min_value()) { - min = 0; - max = 0; - } - assert!(min <= max, "discriminant range is {}...{}", min, max); - let (min_ity, signed) = Integer::repr_discr(tcx, ty, &def.repr, min, max); - - let mut align = dl.aggregate_align; - let mut size = Size::ZERO; - - // We're interested in the smallest alignment, so start large. - let mut start_align = Align::from_bytes(256).unwrap(); - assert_eq!(Integer::for_align(dl, start_align), None); + let mut prefix_align = ity.align(dl).abi; + let align = align.max(AbiAndPrefAlign::new(prefix_align)); // repr(C) on an enum tells us to make a (tag, union) layout, // so we need to grow the prefix alignment to be at least // the alignment of the union. (This value is used both for // determining the alignment of the overall enum, and the // determining the alignment of the payload after the tag.) - let mut prefix_align = min_ity.align(dl).abi; if def.repr.c() { - for fields in &variants { - for field in fields { - prefix_align = prefix_align.max(field.align.abi); - } - } + prefix_align = align.abi; } + let mut size = Size::ZERO; + // Create the set of structs that represent each variant. - let mut layout_variants = variants.iter_enumerated().map(|(i, field_layouts)| { + let layout_variants = variants.iter_enumerated().map(|(i, field_layouts)| { let mut st = self.univariant_uninterned(ty, &field_layouts, - &def.repr, StructKind::Prefixed(min_ity.size(), prefix_align))?; + &def.repr, StructKind::Prefixed(ity.size(), prefix_align))?; st.variants = Variants::Single { index: i }; - // Find the first field we can't move later - // to make room for a larger discriminant. - for field in st.fields.index_by_increasing_offset().map(|j| field_layouts[j]) { - if !field.is_zst() || field.align.abi.bytes() != 1 { - start_align = start_align.min(field.align.abi); - break; - } - } size = cmp::max(size, st.size); - align = align.max(st.align); + assert!(st.align.abi <= align.abi && st.align.pref <= align.pref); Ok(st) }).collect::, _>>()?; @@ -1094,66 +1130,6 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> { return Err(LayoutError::SizeOverflow(ty)); } - let typeck_ity = Integer::from_attr(dl, def.repr.discr_type()); - if typeck_ity < min_ity { - // It is a bug if Layout decided on a greater discriminant size than typeck for - // some reason at this point (based on values discriminant can take on). Mostly - // because this discriminant will be loaded, and then stored into variable of - // type calculated by typeck. Consider such case (a bug): typeck decided on - // byte-sized discriminant, but layout thinks we need a 16-bit to store all - // discriminant values. That would be a bug, because then, in codegen, in order - // to store this 16-bit discriminant into 8-bit sized temporary some of the - // space necessary to represent would have to be discarded (or layout is wrong - // on thinking it needs 16 bits) - bug!("layout decided on a larger discriminant type ({:?}) than typeck ({:?})", - min_ity, typeck_ity); - // However, it is fine to make discr type however large (as an optimisation) - // after this point – we’ll just truncate the value we load in codegen. - } - - // Check to see if we should use a different type for the - // discriminant. We can safely use a type with the same size - // as the alignment of the first field of each variant. - // We increase the size of the discriminant to avoid LLVM copying - // padding when it doesn't need to. This normally causes unaligned - // load/stores and excessive memcpy/memset operations. By using a - // bigger integer size, LLVM can be sure about its contents and - // won't be so conservative. - - // Use the initial field alignment - let mut ity = if def.repr.c() || def.repr.int.is_some() { - min_ity - } else { - Integer::for_align(dl, start_align).unwrap_or(min_ity) - }; - - // If the alignment is not larger than the chosen discriminant size, - // don't use the alignment as the final size. - if ity <= min_ity { - ity = min_ity; - } else { - // Patch up the variants' first few fields. - let old_ity_size = min_ity.size(); - let new_ity_size = ity.size(); - for variant in &mut layout_variants { - match variant.fields { - FieldPlacement::Arbitrary { ref mut offsets, .. } => { - for i in offsets { - if *i <= old_ity_size { - assert_eq!(*i, old_ity_size); - *i = new_ity_size; - } - } - // We might be making the struct larger. - if variant.size <= old_ity_size { - variant.size = new_ity_size; - } - } - _ => bug!() - } - } - } - let tag_mask = !0u128 >> (128 - ity.size().bits()); let tag = Scalar { value: Int(ity, signed), diff --git a/src/librustc_target/abi/mod.rs b/src/librustc_target/abi/mod.rs index dafa866117681..a4eaa34388b96 100644 --- a/src/librustc_target/abi/mod.rs +++ b/src/librustc_target/abi/mod.rs @@ -533,6 +533,16 @@ impl Integer { } I8 } + + /// Finds the largest integer with the given size or less. + pub fn approximate_size(wanted: Size) -> Option { + for &candidate in &[I64, I32, I16, I8] { + if wanted >= candidate.size() { + return Some(candidate); + } + } + None + } } diff --git a/src/test/codegen/align-enum.rs b/src/test/codegen/align-enum.rs index 4241fcea8047d..326f5c9f60698 100644 --- a/src/test/codegen/align-enum.rs +++ b/src/test/codegen/align-enum.rs @@ -9,7 +9,7 @@ pub enum Align64 { A(u32), B(u32), } -// CHECK: %Align64 = type { [0 x i32], i32, [15 x i32] } +// CHECK: %Align64 = type { [0 x i64], i64, [7 x i64] } pub struct Nested64 { a: u8, diff --git a/src/test/codegen/align-struct.rs b/src/test/codegen/align-struct.rs index c0d6a0c80e1cf..d25b42cac227a 100644 --- a/src/test/codegen/align-struct.rs +++ b/src/test/codegen/align-struct.rs @@ -26,8 +26,9 @@ pub enum Enum64 { A(Align64), B(i32), } -// CHECK: %Enum64 = type { [0 x i32], i32, [31 x i32] } +// CHECK: %Enum64 = type { [0 x i64], i64, [15 x i64] } // CHECK: %"Enum64::A" = type { [8 x i64], %Align64, [0 x i64] } +// CHECK: %"Enum64::B" = type { [2 x i32], i32, [1 x i32] } // CHECK-LABEL: @align64 #[no_mangle] @@ -71,3 +72,11 @@ pub fn enum64(a: Align64) -> Enum64 { let e64 = Enum64::A(a); e64 } + +// CHECK-LABEL: @enum64_b +#[no_mangle] +pub fn enum64_b(b: i32) -> Enum64 { +// CHECK: %e64 = alloca %Enum64, align 64 + let e64 = Enum64::B(b); + e64 +} diff --git a/src/test/ui/print_type_sizes/niche-filling.rs b/src/test/ui/print_type_sizes/niche-filling.rs index d9845fd6d70cc..4675b0f3c0d51 100644 --- a/src/test/ui/print_type_sizes/niche-filling.rs +++ b/src/test/ui/print_type_sizes/niche-filling.rs @@ -27,7 +27,7 @@ impl Default for MyOption { pub enum EmbeddedDiscr { None, - Record { pre: u8, val: NonZeroU32, post: u16 }, + Record { pre1: u8, pre2: u8, val: NonZeroU32, post: u16 }, } impl Default for EmbeddedDiscr { @@ -53,6 +53,15 @@ impl Default for NestedNonZero { } } +pub enum BadNiche { + None, + Record { pre: u8, val: NonZeroU32, post: u16 }, +} + +impl Default for BadNiche { + fn default() -> Self { BadNiche::None } +} + pub enum Enum4 { One(A), Two(B), @@ -74,6 +83,7 @@ fn start(_: isize, _: *const *const u8) -> isize { let _x: MyOption = Default::default(); let _y: EmbeddedDiscr = Default::default(); let _z: MyOption = Default::default(); + let _w: BadNiche = Default::default(); let _a: MyOption = Default::default(); let _b: MyOption = Default::default(); let _c: MyOption = Default::default(); diff --git a/src/test/ui/print_type_sizes/niche-filling.stdout b/src/test/ui/print_type_sizes/niche-filling.stdout index 301edc0d086b1..1865ab3f2e8cb 100644 --- a/src/test/ui/print_type_sizes/niche-filling.stdout +++ b/src/test/ui/print_type_sizes/niche-filling.stdout @@ -7,13 +7,20 @@ print-type-size type: `MyOption`: 12 bytes, alignment: 4 bytes print-type-size variant `Some`: 12 bytes print-type-size field `.0`: 12 bytes print-type-size variant `None`: 0 bytes -print-type-size type: `EmbeddedDiscr`: 8 bytes, alignment: 4 bytes +print-type-size type: `BadNiche`: 8 bytes, alignment: 4 bytes +print-type-size discriminant: 1 bytes print-type-size variant `Record`: 7 bytes +print-type-size field `.pre`: 1 bytes +print-type-size field `.post`: 2 bytes +print-type-size field `.val`: 4 bytes +print-type-size variant `None`: 0 bytes +print-type-size type: `EmbeddedDiscr`: 8 bytes, alignment: 4 bytes +print-type-size variant `Record`: 8 bytes print-type-size field `.val`: 4 bytes print-type-size field `.post`: 2 bytes -print-type-size field `.pre`: 1 bytes +print-type-size field `.pre1`: 1 bytes +print-type-size field `.pre2`: 1 bytes print-type-size variant `None`: 0 bytes -print-type-size end padding: 1 bytes print-type-size type: `MyOption>`: 8 bytes, alignment: 4 bytes print-type-size discriminant: 4 bytes print-type-size variant `Some`: 4 bytes diff --git a/src/test/ui/print_type_sizes/padding.stdout b/src/test/ui/print_type_sizes/padding.stdout index 9afdf76245df7..f948f7c6b5c46 100644 --- a/src/test/ui/print_type_sizes/padding.stdout +++ b/src/test/ui/print_type_sizes/padding.stdout @@ -1,21 +1,19 @@ print-type-size type: `E1`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes +print-type-size discriminant: 4 bytes +print-type-size variant `A`: 8 bytes print-type-size field `.1`: 1 bytes -print-type-size padding: 2 bytes +print-type-size padding: 3 bytes print-type-size field `.0`: 4 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes print-type-size type: `E2`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes +print-type-size discriminant: 4 bytes +print-type-size variant `A`: 8 bytes print-type-size field `.0`: 1 bytes -print-type-size padding: 2 bytes +print-type-size padding: 3 bytes print-type-size field `.1`: 4 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes print-type-size type: `S`: 8 bytes, alignment: 4 bytes print-type-size field `.g`: 4 bytes print-type-size field `.a`: 1 bytes diff --git a/src/test/ui/print_type_sizes/repr-align.stdout b/src/test/ui/print_type_sizes/repr-align.stdout index 33671bd8e14bc..150bb5fd136a1 100644 --- a/src/test/ui/print_type_sizes/repr-align.stdout +++ b/src/test/ui/print_type_sizes/repr-align.stdout @@ -1,7 +1,7 @@ print-type-size type: `E`: 32 bytes, alignment: 16 bytes -print-type-size discriminant: 4 bytes -print-type-size variant `B`: 28 bytes -print-type-size padding: 12 bytes +print-type-size discriminant: 8 bytes +print-type-size variant `B`: 24 bytes +print-type-size padding: 8 bytes print-type-size field `.0`: 16 bytes, alignment: 16 bytes print-type-size variant `A`: 4 bytes print-type-size field `.0`: 4 bytes