diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 68bc0ffce6b5d..8b8d1efbbd2e0 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -980,27 +980,17 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { } } - let tcx = self.tcx; - let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new(); - loop { - if let Some(local) = self.try_as_local(copy_from_local_value, location) { - projection.reverse(); - let place = Place { local, projection: tcx.mk_place_elems(projection.as_slice()) }; - if rvalue.ty(self.local_decls, tcx) == place.ty(self.local_decls, tcx).ty { - self.reused_locals.insert(local); - *rvalue = Rvalue::Use(Operand::Copy(place)); - return Some(copy_from_value); - } - return None; - } else if let Value::Projection(pointer, proj) = *self.get(copy_from_local_value) - && let Some(proj) = self.try_as_place_elem(proj, location) - { - projection.push(proj); - copy_from_local_value = pointer; - } else { - return None; + // Allow introducing places with non-constant offsets, as those are still better than + // reconstructing an aggregate. + if let Some(place) = self.try_as_place(copy_from_local_value, location, true) { + if rvalue.ty(self.local_decls, self.tcx) == place.ty(self.local_decls, self.tcx).ty { + self.reused_locals.insert(place.local); + *rvalue = Rvalue::Use(Operand::Copy(place)); + return Some(copy_from_local_value); } } + + None } fn simplify_aggregate( @@ -1672,14 +1662,14 @@ fn op_to_prop_const<'tcx>( } impl<'tcx> VnState<'_, 'tcx> { - /// If either [`Self::try_as_constant`] as [`Self::try_as_local`] succeeds, + /// If either [`Self::try_as_constant`] as [`Self::try_as_place`] succeeds, /// returns that result as an [`Operand`]. fn try_as_operand(&mut self, index: VnIndex, location: Location) -> Option> { if let Some(const_) = self.try_as_constant(index) { Some(Operand::Constant(Box::new(const_))) - } else if let Some(local) = self.try_as_local(index, location) { - self.reused_locals.insert(local); - Some(Operand::Copy(local.into())) + } else if let Some(place) = self.try_as_place(index, location, false) { + self.reused_locals.insert(place.local); + Some(Operand::Copy(place)) } else { None } @@ -1712,6 +1702,35 @@ impl<'tcx> VnState<'_, 'tcx> { Some(ConstOperand { span: DUMMY_SP, user_ty: None, const_ }) } + /// Construct a place which holds the same value as `index` and for which all locals strictly + /// dominate `loc`. If you used this place, add its base local to `reused_locals` to remove + /// storage statements. + #[instrument(level = "trace", skip(self), ret)] + fn try_as_place( + &mut self, + mut index: VnIndex, + loc: Location, + allow_complex_projection: bool, + ) -> Option> { + let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new(); + loop { + if let Some(local) = self.try_as_local(index, loc) { + projection.reverse(); + let place = + Place { local, projection: self.tcx.mk_place_elems(projection.as_slice()) }; + return Some(place); + } else if let Value::Projection(pointer, proj) = *self.get(index) + && (allow_complex_projection || proj.is_stable_offset()) + && let Some(proj) = self.try_as_place_elem(proj, loc) + { + projection.push(proj); + index = pointer; + } else { + return None; + } + } + } + /// If there is a local which is assigned `index`, and its assignment strictly dominates `loc`, /// return it. If you used this local, add it to `reused_locals` to remove storage statements. fn try_as_local(&mut self, index: VnIndex, loc: Location) -> Option { @@ -1762,11 +1781,12 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> { if let Some(value) = value { if let Some(const_) = self.try_as_constant(value) { *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_))); - } else if let Some(local) = self.try_as_local(value, location) - && *rvalue != Rvalue::Use(Operand::Move(local.into())) + } else if let Some(place) = self.try_as_place(value, location, false) + && *rvalue != Rvalue::Use(Operand::Move(place)) + && *rvalue != Rvalue::Use(Operand::Copy(place)) { - *rvalue = Rvalue::Use(Operand::Copy(local.into())); - self.reused_locals.insert(local); + *rvalue = Rvalue::Use(Operand::Copy(place)); + self.reused_locals.insert(place.local); } } } diff --git a/tests/mir-opt/const_prop/address_of_pair.fn0.GVN.diff b/tests/mir-opt/const_prop/address_of_pair.fn0.GVN.diff index e33185f17bcf7..c23afae829f48 100644 --- a/tests/mir-opt/const_prop/address_of_pair.fn0.GVN.diff +++ b/tests/mir-opt/const_prop/address_of_pair.fn0.GVN.diff @@ -31,15 +31,13 @@ (*_3) = const true; _4 = const (); StorageDead(_4); -- StorageLive(_5); -+ nop; + StorageLive(_5); StorageLive(_6); _6 = copy (_2.1: bool); _5 = Not(move _6); StorageDead(_6); _0 = copy _5; -- StorageDead(_5); -+ nop; + StorageDead(_5); StorageDead(_3); StorageDead(_2); return; diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff index 37dd14e6c8968..35eb4fbd106a1 100644 --- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff +++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff @@ -69,8 +69,7 @@ bb0: { StorageLive(_1); -- StorageLive(_2); -+ nop; + StorageLive(_2); StorageLive(_3); StorageLive(_11); StorageLive(_12); @@ -119,8 +118,7 @@ StorageDead(_11); _2 = &_3; _1 = copy _2; -- StorageDead(_2); -+ nop; + StorageDead(_2); StorageLive(_4); - _9 = deref_copy _3; + _9 = copy _3; @@ -141,7 +139,7 @@ StorageLive(_8); _8 = copy _5; - _7 = copy _8 as *mut () (PtrToPtr); -+ _7 = copy _5 as *mut () (PtrToPtr); ++ _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute); StorageDead(_8); StorageDead(_7); - StorageDead(_5); diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff index 6bac680594319..b2085afb71379 100644 --- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff +++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff @@ -35,8 +35,7 @@ bb0: { StorageLive(_1); -- StorageLive(_2); -+ nop; + StorageLive(_2); StorageLive(_3); _3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue]; } @@ -44,8 +43,7 @@ bb1: { _2 = &_3; _1 = copy _2; -- StorageDead(_2); -+ nop; + StorageDead(_2); StorageLive(_4); - _9 = deref_copy _3; + _9 = copy _3; @@ -66,7 +64,7 @@ StorageLive(_8); _8 = copy _5; - _7 = copy _8 as *mut () (PtrToPtr); -+ _7 = copy _5 as *mut () (PtrToPtr); ++ _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute); StorageDead(_8); StorageDead(_7); - StorageDead(_5); diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff index 1cf0f6de011ed..4427a5fcc7deb 100644 --- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff +++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff @@ -69,8 +69,7 @@ bb0: { StorageLive(_1); -- StorageLive(_2); -+ nop; + StorageLive(_2); StorageLive(_3); StorageLive(_11); StorageLive(_12); @@ -119,8 +118,7 @@ StorageDead(_11); _2 = &_3; _1 = copy _2; -- StorageDead(_2); -+ nop; + StorageDead(_2); StorageLive(_4); - _9 = deref_copy _3; + _9 = copy _3; @@ -141,7 +139,7 @@ StorageLive(_8); _8 = copy _5; - _7 = copy _8 as *mut () (PtrToPtr); -+ _7 = copy _5 as *mut () (PtrToPtr); ++ _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute); StorageDead(_8); StorageDead(_7); - StorageDead(_5); diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff index 6bac680594319..b2085afb71379 100644 --- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff +++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff @@ -35,8 +35,7 @@ bb0: { StorageLive(_1); -- StorageLive(_2); -+ nop; + StorageLive(_2); StorageLive(_3); _3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue]; } @@ -44,8 +43,7 @@ bb1: { _2 = &_3; _1 = copy _2; -- StorageDead(_2); -+ nop; + StorageDead(_2); StorageLive(_4); - _9 = deref_copy _3; + _9 = copy _3; @@ -66,7 +64,7 @@ StorageLive(_8); _8 = copy _5; - _7 = copy _8 as *mut () (PtrToPtr); -+ _7 = copy _5 as *mut () (PtrToPtr); ++ _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute); StorageDead(_8); StorageDead(_7); - StorageDead(_5); diff --git a/tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-abort.diff b/tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-abort.diff new file mode 100644 index 0000000000000..e2e55304921b2 --- /dev/null +++ b/tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-abort.diff @@ -0,0 +1,58 @@ +- // MIR for `compare_constant_index` before GVN ++ // MIR for `compare_constant_index` after GVN + + fn compare_constant_index(_1: [i32; 1], _2: [i32; 1]) -> std::cmp::Ordering { + debug x => _1; + debug y => _2; + let mut _0: std::cmp::Ordering; + let _3: &i32; + let _4: usize; + let mut _5: bool; + let _6: &i32; + let _7: usize; + let mut _8: bool; + scope 1 (inlined std::cmp::impls::::cmp) { + let mut _9: i32; + let mut _10: i32; + } + + bb0: { +- StorageLive(_4); ++ nop; + _4 = const 0_usize; +- _5 = Lt(copy _4, const 1_usize); +- assert(move _5, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _4) -> [success: bb1, unwind unreachable]; ++ _5 = const true; ++ assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb1, unwind unreachable]; + } + + bb1: { +- _3 = &_1[_4]; ++ _3 = &_1[0 of 1]; + StorageLive(_7); + _7 = const 0_usize; +- _8 = Lt(copy _7, const 1_usize); +- assert(move _8, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _7) -> [success: bb2, unwind unreachable]; ++ _8 = const true; ++ assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb2, unwind unreachable]; + } + + bb2: { +- _6 = &_2[_7]; ++ _6 = &_2[0 of 1]; + StorageLive(_9); +- _9 = copy (*_3); ++ _9 = copy _1[0 of 1]; + StorageLive(_10); +- _10 = copy (*_6); ++ _10 = copy _2[0 of 1]; + _0 = Cmp(move _9, move _10); + StorageDead(_10); + StorageDead(_9); + StorageDead(_7); +- StorageDead(_4); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-unwind.diff b/tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-unwind.diff new file mode 100644 index 0000000000000..60611146a0eec --- /dev/null +++ b/tests/mir-opt/gvn_copy_constant_projection.compare_constant_index.GVN.panic-unwind.diff @@ -0,0 +1,58 @@ +- // MIR for `compare_constant_index` before GVN ++ // MIR for `compare_constant_index` after GVN + + fn compare_constant_index(_1: [i32; 1], _2: [i32; 1]) -> std::cmp::Ordering { + debug x => _1; + debug y => _2; + let mut _0: std::cmp::Ordering; + let _3: &i32; + let _4: usize; + let mut _5: bool; + let _6: &i32; + let _7: usize; + let mut _8: bool; + scope 1 (inlined std::cmp::impls::::cmp) { + let mut _9: i32; + let mut _10: i32; + } + + bb0: { +- StorageLive(_4); ++ nop; + _4 = const 0_usize; +- _5 = Lt(copy _4, const 1_usize); +- assert(move _5, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _4) -> [success: bb1, unwind continue]; ++ _5 = const true; ++ assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb1, unwind continue]; + } + + bb1: { +- _3 = &_1[_4]; ++ _3 = &_1[0 of 1]; + StorageLive(_7); + _7 = const 0_usize; +- _8 = Lt(copy _7, const 1_usize); +- assert(move _8, "index out of bounds: the length is {} but the index is {}", const 1_usize, copy _7) -> [success: bb2, unwind continue]; ++ _8 = const true; ++ assert(const true, "index out of bounds: the length is {} but the index is {}", const 1_usize, const 0_usize) -> [success: bb2, unwind continue]; + } + + bb2: { +- _6 = &_2[_7]; ++ _6 = &_2[0 of 1]; + StorageLive(_9); +- _9 = copy (*_3); ++ _9 = copy _1[0 of 1]; + StorageLive(_10); +- _10 = copy (*_6); ++ _10 = copy _2[0 of 1]; + _0 = Cmp(move _9, move _10); + StorageDead(_10); + StorageDead(_9); + StorageDead(_7); +- StorageDead(_4); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/gvn_copy_constant_projection.rs b/tests/mir-opt/gvn_copy_constant_projection.rs new file mode 100644 index 0000000000000..a08ae0ac7c976 --- /dev/null +++ b/tests/mir-opt/gvn_copy_constant_projection.rs @@ -0,0 +1,18 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY + +use std::cmp::Ordering; +fn compare_constant_index(x: [i32; 1], y: [i32; 1]) -> Ordering { + // CHECK-LABEL: fn compare_constant_index( + // CHECK-NOT: (*{{_.*}}); + // CHECK: [[lhs:_.*]] = copy _1[0 of 1]; + // CHECK-NOT: (*{{_.*}}); + // CHECK: [[rhs:_.*]] = copy _2[0 of 1]; + // CHECK: _0 = Cmp(move [[lhs]], move [[rhs]]); + Ord::cmp(&x[0], &y[0]) +} + +fn main() { + compare_constant_index([1], [2]); +} + +// EMIT_MIR gvn_copy_constant_projection.compare_constant_index.GVN.diff diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff index eb97af1e2845f..e49d7cea28e58 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff @@ -229,7 +229,6 @@ + StorageDead(_24); + StorageLive(_45); + StorageLive(_46); -+ StorageLive(_49); + StorageLive(_51); + StorageLive(_42); + StorageLive(_43); @@ -243,9 +242,11 @@ + _47 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _45 }; + StorageDead(_47); + _44 = &mut ((*_45).0: std::option::Option<()>); ++ StorageLive(_49); + _49 = Option::<()>::None; + _43 = copy ((*_45).0: std::option::Option<()>); + ((*_45).0: std::option::Option<()>) = copy _49; ++ StorageDead(_49); + StorageDead(_44); + StorageLive(_50); + _50 = discriminant(_43); @@ -322,7 +323,6 @@ + _18 = Poll::<()>::Ready(move _42); + StorageDead(_42); + StorageDead(_51); -+ StorageDead(_49); + StorageDead(_46); + StorageDead(_45); + StorageDead(_22); diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff index eb757e0911453..e7aed556f2d71 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff @@ -246,7 +246,6 @@ + StorageDead(_24); + StorageLive(_47); + StorageLive(_48); -+ StorageLive(_51); + StorageLive(_53); + StorageLive(_44); + StorageLive(_45); @@ -260,9 +259,11 @@ + _49 = Pin::<&mut std::future::Ready<()>> { __pointer: copy _47 }; + StorageDead(_49); + _46 = &mut ((*_47).0: std::option::Option<()>); ++ StorageLive(_51); + _51 = Option::<()>::None; + _45 = copy ((*_47).0: std::option::Option<()>); + ((*_47).0: std::option::Option<()>) = copy _51; ++ StorageDead(_51); + StorageDead(_46); + StorageLive(_52); + _52 = discriminant(_45); @@ -363,7 +364,6 @@ + _18 = Poll::<()>::Ready(move _44); + StorageDead(_44); + StorageDead(_53); -+ StorageDead(_51); + StorageDead(_48); + StorageDead(_47); + StorageDead(_22); diff --git a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir index 49314a64c3fc9..8746cb0899166 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir @@ -40,6 +40,7 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { StorageLive(_12); StorageLive(_11); StorageLive(_5); + StorageLive(_6); StorageLive(_7); StorageLive(_3); _3 = copy ((*_1).0: char); @@ -65,6 +66,7 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { _11 = Option::::Some(move _10); StorageDead(_10); StorageDead(_7); + StorageDead(_6); StorageDead(_5); goto -> bb3; } @@ -72,6 +74,7 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { bb2: { _11 = copy _6; StorageDead(_7); + StorageDead(_6); StorageDead(_5); goto -> bb3; } diff --git a/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir index dd2eebc8f4a14..c4d0e318b58dc 100644 --- a/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir @@ -44,8 +44,10 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool { StorageDead(_5); StorageDead(_4); StorageDead(_3); + StorageLive(_8); _8 = copy ((_7 as Break).0: bool); _0 = copy _8; + StorageDead(_8); goto -> bb3; } diff --git a/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir index ea1d164cefafb..44df8b279935b 100644 --- a/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir @@ -44,8 +44,10 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool { StorageDead(_5); StorageDead(_4); StorageDead(_3); + StorageLive(_8); _8 = copy ((_7 as Break).0: bool); _0 = copy _8; + StorageDead(_8); goto -> bb3; }