Skip to content

Commit d33946c

Browse files
compiler-errorscompiler-errors
committed
Inline FnOnce once again
1 parent e081b7b commit d33946c

23 files changed

+391
-116
lines changed

‎compiler/rustc_mir_transform/src/inline.rs

+10-13
Original file line numberDiff line numberDiff line change
@@ -625,13 +625,6 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(
625625
return Err("implementation limitation -- return type mismatch");
626626
}
627627
if callsite.fn_sig.abi() == ExternAbi::RustCall {
628-
// FIXME: Don't inline user-written `extern "rust-call"` functions,
629-
// since this is generally perf-negative on rustc, and we hope that
630-
// LLVM will inline these functions instead.
631-
if callee_body.spread_arg.is_some() {
632-
return Err("user-written rust-call functions");
633-
}
634-
635628
let (self_arg, arg_tuple) = match &args[..] {
636629
[arg_tuple] => (None, arg_tuple),
637630
[self_arg, arg_tuple] => (Some(self_arg), arg_tuple),
@@ -641,18 +634,23 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(
641634
let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx));
642635

643636
let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx);
644-
let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
645-
bug!("Closure arguments are not passed as a tuple");
637+
let arg_tys = if callee_body.spread_arg.is_some() {
638+
std::slice::from_ref(&arg_tuple_ty)
639+
} else {
640+
let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
641+
bug!("Closure arguments are not passed as a tuple");
642+
};
643+
arg_tuple_tys.as_slice()
646644
};
647645

648646
for (arg_ty, input) in
649-
self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter())
647+
self_arg_ty.into_iter().chain(arg_tys.iter().copied()).zip(callee_body.args_iter())
650648
{
651649
let input_type = callee_body.local_decls[input].ty;
652650
if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
653651
trace!(?arg_ty, ?input_type);
654652
debug!("failed to normalize tuple argument type");
655-
return Err("implementation limitation -- arg mismatch");
653+
return Err("implementation limitation");
656654
}
657655
}
658656
} else {
@@ -1059,8 +1057,7 @@ fn make_call_args<'tcx, I: Inliner<'tcx>>(
10591057

10601058
closure_ref_arg.chain(tuple_tmp_args).collect()
10611059
} else {
1062-
// FIXME(edition_2024): switch back to a normal method call.
1063-
<_>::into_iter(args)
1060+
args.into_iter()
10641061
.map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block))
10651062
.collect()
10661063
}

‎tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff

+24-5
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,42 @@
77
let mut _0: ();
88
let mut _3: &mut std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
99
let mut _4: I;
10+
+ scope 1 (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut) {
11+
+ let mut _5: &mut dyn std::ops::FnMut<I, Output = ()>;
12+
+ let mut _6: std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
13+
+ let mut _7: *const dyn std::ops::FnMut<I, Output = ()>;
14+
+ }
1015

1116
bb0: {
1217
StorageLive(_3);
1318
_3 = &mut _1;
1419
StorageLive(_4);
1520
_4 = move _2;
16-
_0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable];
21+
- _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable];
22+
+ StorageLive(_6);
23+
+ StorageLive(_7);
24+
+ StorageLive(_5);
25+
+ _6 = copy (*_3);
26+
+ _7 = copy ((_6.0: std::ptr::Unique<dyn std::ops::FnMut<I, Output = ()>>).0: std::ptr::NonNull<dyn std::ops::FnMut<I, Output = ()>>) as *const dyn std::ops::FnMut<I, Output = ()> (Transmute);
27+
+ _5 = &mut (*_7);
28+
+ _0 = <dyn FnMut<I, Output = ()> as FnMut<I>>::call_mut(move _5, move _4) -> [return: bb2, unwind unreachable];
1729
}
1830

1931
bb1: {
20-
StorageDead(_4);
21-
StorageDead(_3);
22-
drop(_1) -> [return: bb2, unwind unreachable];
32+
- StorageDead(_4);
33+
- StorageDead(_3);
34+
- drop(_1) -> [return: bb2, unwind unreachable];
35+
+ return;
2336
}
2437

2538
bb2: {
26-
return;
39+
- return;
40+
+ StorageDead(_5);
41+
+ StorageDead(_7);
42+
+ StorageDead(_6);
43+
+ StorageDead(_4);
44+
+ StorageDead(_3);
45+
+ drop(_1) -> [return: bb1, unwind unreachable];
2746
}
2847
}
2948

‎tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff

+32-9
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,54 @@
77
let mut _0: ();
88
let mut _3: &mut std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
99
let mut _4: I;
10+
+ scope 1 (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut) {
11+
+ let mut _5: &mut dyn std::ops::FnMut<I, Output = ()>;
12+
+ let mut _6: std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
13+
+ let mut _7: *const dyn std::ops::FnMut<I, Output = ()>;
14+
+ }
1015

1116
bb0: {
1217
StorageLive(_3);
1318
_3 = &mut _1;
1419
StorageLive(_4);
1520
_4 = move _2;
16-
_0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind: bb3];
21+
- _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind: bb3];
22+
+ StorageLive(_6);
23+
+ StorageLive(_7);
24+
+ StorageLive(_5);
25+
+ _6 = copy (*_3);
26+
+ _7 = copy ((_6.0: std::ptr::Unique<dyn std::ops::FnMut<I, Output = ()>>).0: std::ptr::NonNull<dyn std::ops::FnMut<I, Output = ()>>) as *const dyn std::ops::FnMut<I, Output = ()> (Transmute);
27+
+ _5 = &mut (*_7);
28+
+ _0 = <dyn FnMut<I, Output = ()> as FnMut<I>>::call_mut(move _5, move _4) -> [return: bb4, unwind: bb2];
1729
}
1830

1931
bb1: {
20-
StorageDead(_4);
21-
StorageDead(_3);
22-
drop(_1) -> [return: bb2, unwind: bb4];
32+
- StorageDead(_4);
33+
- StorageDead(_3);
34+
- drop(_1) -> [return: bb2, unwind: bb4];
35+
+ return;
2336
}
2437

25-
bb2: {
26-
return;
38+
- bb2: {
39+
- return;
40+
+ bb2 (cleanup): {
41+
+ drop(_1) -> [return: bb3, unwind terminate(cleanup)];
2742
}
2843

2944
bb3 (cleanup): {
30-
drop(_1) -> [return: bb4, unwind terminate(cleanup)];
45+
- drop(_1) -> [return: bb4, unwind terminate(cleanup)];
46+
+ resume;
3147
}
3248

33-
bb4 (cleanup): {
34-
resume;
49+
- bb4 (cleanup): {
50+
- resume;
51+
+ bb4: {
52+
+ StorageDead(_5);
53+
+ StorageDead(_7);
54+
+ StorageDead(_6);
55+
+ StorageDead(_4);
56+
+ StorageDead(_3);
57+
+ drop(_1) -> [return: bb1, unwind: bb3];
3558
}
3659
}
3760

‎tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@ use std::marker::Tuple;
88
// EMIT_MIR dont_ice_on_generic_rust_call.call.Inline.diff
99
pub fn call<I: Tuple>(mut mock: Box<dyn FnMut<I, Output = ()>>, input: I) {
1010
// CHECK-LABEL: fn call(
11-
// CHECK-NOT: inlined
11+
// CHECK: (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut)
1212
mock.call_mut(input)
1313
}

‎tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff

+26-6
Original file line numberDiff line numberDiff line change
@@ -7,26 +7,46 @@
77
let _2: ();
88
let mut _3: &std::boxed::Box<dyn std::ops::Fn(i32)>;
99
let mut _4: (i32,);
10+
+ scope 1 (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call) {
11+
+ let mut _5: &dyn std::ops::Fn(i32);
12+
+ let mut _6: std::boxed::Box<dyn std::ops::Fn(i32)>;
13+
+ let mut _7: *const dyn std::ops::Fn(i32);
14+
+ }
1015

1116
bb0: {
1217
StorageLive(_2);
1318
StorageLive(_3);
1419
_3 = &_1;
1520
StorageLive(_4);
1621
_4 = (const 1_i32,);
17-
_2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable];
22+
- _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable];
23+
+ StorageLive(_6);
24+
+ StorageLive(_7);
25+
+ StorageLive(_5);
26+
+ _6 = copy (*_3);
27+
+ _7 = copy ((_6.0: std::ptr::Unique<dyn std::ops::Fn(i32)>).0: std::ptr::NonNull<dyn std::ops::Fn(i32)>) as *const dyn std::ops::Fn(i32) (Transmute);
28+
+ _5 = &(*_7);
29+
+ _2 = <dyn Fn(i32) as Fn<(i32,)>>::call(move _5, move _4) -> [return: bb2, unwind unreachable];
1830
}
1931

2032
bb1: {
33+
+ return;
34+
+ }
35+
+
36+
+ bb2: {
37+
+ StorageDead(_5);
38+
+ StorageDead(_7);
39+
+ StorageDead(_6);
2140
StorageDead(_4);
2241
StorageDead(_3);
2342
StorageDead(_2);
2443
_0 = const ();
25-
drop(_1) -> [return: bb2, unwind unreachable];
26-
}
27-
28-
bb2: {
29-
return;
44+
- drop(_1) -> [return: bb2, unwind unreachable];
45+
- }
46+
-
47+
- bb2: {
48+
- return;
49+
+ drop(_1) -> [return: bb1, unwind unreachable];
3050
}
3151
}
3252

‎tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff

+36-11
Original file line numberDiff line numberDiff line change
@@ -7,34 +7,59 @@
77
let _2: ();
88
let mut _3: &std::boxed::Box<dyn std::ops::Fn(i32)>;
99
let mut _4: (i32,);
10+
+ scope 1 (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call) {
11+
+ let mut _5: &dyn std::ops::Fn(i32);
12+
+ let mut _6: std::boxed::Box<dyn std::ops::Fn(i32)>;
13+
+ let mut _7: *const dyn std::ops::Fn(i32);
14+
+ }
1015

1116
bb0: {
1217
StorageLive(_2);
1318
StorageLive(_3);
1419
_3 = &_1;
1520
StorageLive(_4);
1621
_4 = (const 1_i32,);
17-
_2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind: bb3];
22+
- _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind: bb3];
23+
+ StorageLive(_6);
24+
+ StorageLive(_7);
25+
+ StorageLive(_5);
26+
+ _6 = copy (*_3);
27+
+ _7 = copy ((_6.0: std::ptr::Unique<dyn std::ops::Fn(i32)>).0: std::ptr::NonNull<dyn std::ops::Fn(i32)>) as *const dyn std::ops::Fn(i32) (Transmute);
28+
+ _5 = &(*_7);
29+
+ _2 = <dyn Fn(i32) as Fn<(i32,)>>::call(move _5, move _4) -> [return: bb4, unwind: bb2];
1830
}
1931

2032
bb1: {
21-
StorageDead(_4);
22-
StorageDead(_3);
23-
StorageDead(_2);
24-
_0 = const ();
25-
drop(_1) -> [return: bb2, unwind: bb4];
33+
- StorageDead(_4);
34+
- StorageDead(_3);
35+
- StorageDead(_2);
36+
- _0 = const ();
37+
- drop(_1) -> [return: bb2, unwind: bb4];
38+
+ return;
2639
}
2740

28-
bb2: {
29-
return;
41+
- bb2: {
42+
- return;
43+
+ bb2 (cleanup): {
44+
+ drop(_1) -> [return: bb3, unwind terminate(cleanup)];
3045
}
3146

3247
bb3 (cleanup): {
33-
drop(_1) -> [return: bb4, unwind terminate(cleanup)];
48+
- drop(_1) -> [return: bb4, unwind terminate(cleanup)];
49+
+ resume;
3450
}
3551

36-
bb4 (cleanup): {
37-
resume;
52+
- bb4 (cleanup): {
53+
- resume;
54+
+ bb4: {
55+
+ StorageDead(_5);
56+
+ StorageDead(_7);
57+
+ StorageDead(_6);
58+
+ StorageDead(_4);
59+
+ StorageDead(_3);
60+
+ StorageDead(_2);
61+
+ _0 = const ();
62+
+ drop(_1) -> [return: bb1, unwind: bb3];
3863
}
3964
}
4065

‎tests/mir-opt/inline/inline_box_fn.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@
55
// EMIT_MIR inline_box_fn.call.Inline.diff
66
fn call(x: Box<dyn Fn(i32)>) {
77
// CHECK-LABEL: fn call(
8-
// CHECK-NOT: inlined
8+
// CHECK: (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call)
99
x(1);
1010
}

‎tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff

+12-1
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,15 @@
55
let mut _0: ();
66
let _1: ();
77
+ let mut _2: fn() {f};
8+
+ let mut _4: ();
89
+ scope 1 (inlined call::<fn() {f}>) {
910
+ debug f => _2;
1011
+ let _3: ();
12+
+ scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
13+
+ scope 3 (inlined f) {
14+
+ let _5: ();
15+
+ }
16+
+ }
1117
+ }
1218

1319
bb0: {
@@ -16,10 +22,15 @@
1622
+ StorageLive(_2);
1723
+ _2 = f;
1824
+ StorageLive(_3);
19-
+ _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind unreachable];
25+
+ StorageLive(_4);
26+
+ _4 = const ();
27+
+ StorageLive(_5);
28+
+ _5 = call::<fn() {f}>(f) -> [return: bb1, unwind unreachable];
2029
}
2130

2231
bb1: {
32+
+ StorageDead(_5);
33+
+ StorageDead(_4);
2334
+ StorageDead(_3);
2435
+ StorageDead(_2);
2536
StorageDead(_1);

‎tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff

+12-1
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,15 @@
55
let mut _0: ();
66
let _1: ();
77
+ let mut _2: fn() {f};
8+
+ let mut _4: ();
89
+ scope 1 (inlined call::<fn() {f}>) {
910
+ debug f => _2;
1011
+ let _3: ();
12+
+ scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
13+
+ scope 3 (inlined f) {
14+
+ let _5: ();
15+
+ }
16+
+ }
1117
+ }
1218

1319
bb0: {
@@ -16,10 +22,15 @@
1622
+ StorageLive(_2);
1723
+ _2 = f;
1824
+ StorageLive(_3);
19-
+ _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind continue];
25+
+ StorageLive(_4);
26+
+ _4 = const ();
27+
+ StorageLive(_5);
28+
+ _5 = call::<fn() {f}>(f) -> [return: bb1, unwind continue];
2029
}
2130

2231
bb1: {
32+
+ StorageDead(_5);
33+
+ StorageDead(_4);
2334
+ StorageDead(_3);
2435
+ StorageDead(_2);
2536
StorageDead(_1);

0 commit comments

Comments
 (0)