Skip to content

Commit 63dcb19

Browse files
committed
Inline non-local #[inline(always)] function in incremental mode
1 parent a11235d commit 63dcb19

File tree

4 files changed

+92
-16
lines changed

4 files changed

+92
-16
lines changed

compiler/rustc_mir_transform/src/inline.rs

+89-14
Original file line numberDiff line numberDiff line change
@@ -42,19 +42,7 @@ struct CallSite<'tcx> {
4242

4343
impl<'tcx> MirPass<'tcx> for Inline {
4444
fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
45-
if let Some(enabled) = sess.opts.unstable_opts.inline_mir {
46-
return enabled;
47-
}
48-
49-
match sess.mir_opt_level() {
50-
0 | 1 => false,
51-
2 => {
52-
(sess.opts.optimize == OptLevel::Default
53-
|| sess.opts.optimize == OptLevel::Aggressive)
54-
&& sess.opts.incremental == None
55-
}
56-
_ => true,
57-
}
45+
InlinerConfig::new(sess).is_ok()
5846
}
5947

6048
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -69,6 +57,67 @@ impl<'tcx> MirPass<'tcx> for Inline {
6957
}
7058
}
7159

60+
impl Inline {
61+
pub fn is_enabled_and_needs_mir_inliner_callees(sess: &rustc_session::Session) -> bool {
62+
match InlinerConfig::new(sess) {
63+
Ok(config) => config.inline_local_fns,
64+
Err(_) => false,
65+
}
66+
}
67+
}
68+
69+
struct InlinerConfig {
70+
/// Inline functions with `#[inline(always)]` attribute
71+
inline_fns_with_inline_always_hint: bool,
72+
/// Inline functions with `#[inline]` attribute
73+
inline_fns_with_inline_hint: bool,
74+
/// Inline functions without `#[inline]` attribute
75+
/// Inline functions with `#[inline(always)]` attribute
76+
inline_fns_without_hint: bool,
77+
/// Inline function from current crate (much heavier during incremental compilation)
78+
inline_local_fns: bool,
79+
}
80+
81+
impl InlinerConfig {
82+
fn new(sess: &rustc_session::Session) -> Result<InlinerConfig, InliningIsDisabled> {
83+
match sess.opts.unstable_opts.inline_mir {
84+
Some(true) => return Ok(InlinerConfig::full()),
85+
Some(false) => return Err(InliningIsDisabled),
86+
None => {}
87+
}
88+
match sess.mir_opt_level() {
89+
0 | 1 => Err(InliningIsDisabled),
90+
2 => {
91+
let optimize = sess.opts.optimize;
92+
if optimize == OptLevel::Default || optimize == OptLevel::Aggressive {
93+
let is_non_incremental = sess.opts.incremental == None;
94+
Ok(InlinerConfig {
95+
inline_fns_with_inline_always_hint: true,
96+
inline_fns_with_inline_hint: is_non_incremental,
97+
inline_fns_without_hint: is_non_incremental,
98+
inline_local_fns: is_non_incremental,
99+
})
100+
} else {
101+
Err(InliningIsDisabled)
102+
}
103+
}
104+
_ => Ok(InlinerConfig::full()),
105+
}
106+
}
107+
108+
fn full() -> InlinerConfig {
109+
InlinerConfig {
110+
inline_fns_with_inline_always_hint: true,
111+
inline_fns_with_inline_hint: true,
112+
inline_fns_without_hint: true,
113+
inline_local_fns: true,
114+
}
115+
}
116+
}
117+
118+
#[derive(Debug)]
119+
struct InliningIsDisabled;
120+
72121
fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
73122
let def_id = body.source.def_id().expect_local();
74123

@@ -91,6 +140,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
91140
let mut this = Inliner {
92141
tcx,
93142
param_env,
143+
config: InlinerConfig::new(tcx.sess).unwrap_or_else(|_| InlinerConfig::full()),
94144
codegen_fn_attrs: tcx.codegen_fn_attrs(def_id),
95145
history: Vec::new(),
96146
changed: false,
@@ -103,6 +153,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
103153
struct Inliner<'tcx> {
104154
tcx: TyCtxt<'tcx>,
105155
param_env: ParamEnv<'tcx>,
156+
config: InlinerConfig,
106157
/// Caller codegen attributes.
107158
codegen_fn_attrs: &'tcx CodegenFnAttrs,
108159
/// Stack of inlined instances.
@@ -352,11 +403,35 @@ impl<'tcx> Inliner<'tcx> {
352403
if let InlineAttr::Never = callee_attrs.inline {
353404
return Err("never inline hint");
354405
}
406+
match callee_attrs.inline {
407+
InlineAttr::Never => return Err("never inline hint"),
408+
InlineAttr::Always => {
409+
if !self.config.inline_fns_with_inline_always_hint {
410+
return Err("inliner is configured to ignore #[inline(always)] functions");
411+
}
412+
}
413+
InlineAttr::Hint => {
414+
if !self.config.inline_fns_with_inline_hint {
415+
return Err("inliner is configured to ignore #[inline] functions");
416+
}
417+
}
418+
_ => {
419+
if !self.config.inline_fns_without_hint {
420+
return Err("inliner is configured to ignore functions without #[inline]");
421+
}
422+
}
423+
}
424+
425+
let callee_is_local = callsite.callee.def_id().is_local();
426+
427+
if callee_is_local && !self.config.inline_local_fns {
428+
return Err("inliner is configured to ignore local functions");
429+
}
355430

356431
// Only inline local functions if they would be eligible for cross-crate
357432
// inlining. This is to ensure that the final crate doesn't have MIR that
358433
// reference unexported symbols
359-
if callsite.callee.def_id().is_local() {
434+
if callee_is_local {
360435
let is_generic = callsite.callee.substs.non_erasable_generics().next().is_some();
361436
if !is_generic && !callee_attrs.requests_inline() {
362437
return Err("not exported");

compiler/rustc_mir_transform/src/lib.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -388,7 +388,7 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
388388
let is_fn_like = tcx.def_kind(def).is_fn_like();
389389
if is_fn_like {
390390
// Do not compute the mir call graph without said call graph actually being used.
391-
if inline::Inline.is_enabled(&tcx.sess) {
391+
if inline::Inline::is_enabled_and_needs_mir_inliner_callees(&tcx.sess) {
392392
tcx.ensure_with_value().mir_inliner_callees(ty::InstanceDef::Item(def.to_def_id()));
393393
}
394394
}

tests/codegen-units/partitioning/inlining-from-extern-crate.rs

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
// incremental
55
// compile-flags:-Zprint-mono-items=lazy
66
// compile-flags:-Zinline-in-all-cgus
7+
// compile-flags:-Zinline_mir=false
78

89
#![crate_type="lib"]
910

tests/incremental/hashes/function_interfaces.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ pub fn return_impl_trait() -> i32 {
302302
}
303303

304304
#[cfg(not(any(cfail1,cfail4)))]
305-
#[rustc_clean(cfg = "cfail2", except = "hir_owner, hir_owner_nodes, typeck, fn_sig, optimized_mir")]
305+
#[rustc_clean(cfg = "cfail2", except = "hir_owner, hir_owner_nodes, typeck, fn_sig")]
306306
#[rustc_clean(cfg = "cfail3")]
307307
#[rustc_clean(cfg = "cfail5", except = "hir_owner, hir_owner_nodes, typeck, fn_sig, optimized_mir")]
308308
#[rustc_clean(cfg = "cfail6")]

0 commit comments

Comments
 (0)