diff --git a/crates/rue-codegen/src/aarch64/regalloc.rs b/crates/rue-codegen/src/aarch64/regalloc.rs index 221de37a6..171c76289 100644 --- a/crates/rue-codegen/src/aarch64/regalloc.rs +++ b/crates/rue-codegen/src/aarch64/regalloc.rs @@ -1352,6 +1352,12 @@ impl RegAllocBackend for Aarch64Backend { RegisterClasses { caller_saved: CALLER_SAVED_REGS, callee_saved: CALLEE_SAVED_REGS, + // AArch64 instructions are a fixed four bytes and encode every + // general register in the same five-bit field, so no allocatable + // register is cheaper to address than another and the RUE-1227 + // preference has nothing to trade. Reusing a callee-saved register + // in place of a caller-saved one here would only add pressure. + compact_callee_saved: &[], } } diff --git a/crates/rue-codegen/src/liveness.rs b/crates/rue-codegen/src/liveness.rs index eb6504b1b..e32776b36 100644 --- a/crates/rue-codegen/src/liveness.rs +++ b/crates/rue-codegen/src/liveness.rs @@ -492,6 +492,32 @@ fn compute_dataflow( /// number of live bits, whereas this path stays linear (RUE-302). /// * **Has back-edges (loops):** loop-carried values are live past their textual /// last use, so we fall back to the exact `live_in`/`live_out` scan. +/// +/// # A range is a textual interval, not liveness +/// +/// Either way the result is a single `[start, end]` span of *instruction +/// indices*, and every index between the endpoints is inside it. That is not +/// the same claim as "the value is live at each of those instructions": a range +/// is a contiguous approximation of a set that dataflow may know to have holes. +/// Allocation is built on the interval reading — `LiveRange::overlaps` decides +/// interference, and [`ClobberIndex`](crate::regalloc::ClobberIndex) asks +/// whether anything in the span destroys a register — so anything that makes +/// the two readings disagree is a miscompile, not an imprecision. +/// +/// This is why refining "the value is not really live there" needs the two-part +/// treatment RUE-1224 gave the non-returning trap calls, and not just the +/// dataflow half. Removing a call's successors empties its live-out and can +/// shorten a range that *ends* at the call, but a range that merely *spans* the +/// call does not shrink at all: its endpoints are a def before and a use after, +/// and every index between them stays in the interval no matter what +/// `live_out` says. RUE-1224 therefore also had to exclude those calls from +/// `ClobberIndex`, because the clobber remained inside the interval and would +/// otherwise have kept disqualifying the value from a caller-saved register. +/// +/// So a future refinement here should expect the same shape: model the fact in +/// the dataflow *and* teach every consumer that reads the range as a dense +/// interval about the exclusion. Changing only one of the two silently changes +/// what allocation believes about register lifetimes. fn build_live_ranges( num_insts: usize, vreg_count: u32, diff --git a/crates/rue-codegen/src/regalloc.rs b/crates/rue-codegen/src/regalloc.rs index 8560b0b5e..06fdd4600 100644 --- a/crates/rue-codegen/src/regalloc.rs +++ b/crates/rue-codegen/src/regalloc.rs @@ -568,6 +568,18 @@ pub struct RegisterClasses<'a, Reg> { pub caller_saved: &'a [Reg], /// Tried next, in order; saved by the prologue when used. pub callee_saved: &'a [Reg], + /// The subset of `callee_saved` that instructions address at least as + /// cheaply as any register in `caller_saved`. + /// + /// These are the only registers worth taking *back* from the caller-saved + /// class once their prologue save is already paid for (RUE-1227): reusing a + /// callee-saved register that encodes no better than the caller-saved + /// candidate trades away a free register for nothing, and adds pressure + /// besides. On x86-64 this is `rbx` alone — every other allocatable + /// register is an extended one whose byte and dword forms need a REX + /// prefix, exactly as `r11`'s do. On a fixed-width instruction set it is + /// empty, and the preference below never fires. + pub compact_callee_saved: &'a [Reg], } impl<'a, Reg: Copy + Eq> RegisterClasses<'a, Reg> { @@ -579,6 +591,7 @@ impl<'a, Reg: Copy + Eq> RegisterClasses<'a, Reg> { Self { caller_saved: &[], callee_saved: regs, + compact_callee_saved: &[], } } @@ -1734,21 +1747,42 @@ pub fn linear_scan_with_cost_model_and_debug( /// Pick a physical register for an interval covering `range`, or report that /// none is free. /// -/// Caller-saved registers come first: an interval that no instruction clobbers -/// while it is live costs nothing to keep in one, and every callee-saved -/// register it leaves alone is one the prologue does not have to save -/// (RUE-1146). Callee-saved registers follow, in their declared order. +/// Caller-saved registers come before callee-saved ones: an interval that no +/// instruction clobbers while it is live costs nothing to keep in one, and +/// every callee-saved register it leaves alone is one the prologue does not +/// have to save (RUE-1146). +/// +/// Ahead of both sits one narrow exception, the RUE-1227 tiebreak: a register +/// that is both in [`RegisterClasses::compact_callee_saved`] and in `sunk` — +/// the callee-saved registers this function's prologue already saves. Against a +/// *fresh* callee-saved register the caller-saved candidate wins on the save it +/// avoids, but against one whose save is already paid for it has nothing left +/// to offer and a worse encoding, so preferring the sunk register is free. +/// +/// The exception cannot enlarge the save set: it only ever hands out a register +/// already in it. It can still *shift* which registers end up saved, because +/// occupying a sunk register denies it to a later call-crossing interval that +/// then reaches for a fresh one. Ruling that out is [`accept_reuse_pass`]'s +/// job, not this function's. fn pick_free_register( classes: RegisterClasses<'_, Reg>, clobbers: &ClobberIndex, used: &HashSet, + sunk: &[Reg], range: &LiveRange, ) -> Option { classes - .caller_saved + .compact_callee_saved .iter() .copied() - .find(|®| !used.contains(®) && !clobbers.is_clobbered_during(reg, range)) + .find(|®| sunk.contains(®) && !used.contains(®)) + .or_else(|| { + classes + .caller_saved + .iter() + .copied() + .find(|®| !used.contains(®) && !clobbers.is_clobbered_during(reg, range)) + }) .or_else(|| { classes .callee_saved @@ -1775,14 +1809,64 @@ fn register_survives_range( classes.is_callee_saved(reg) || !clobbers.is_clobbered_during(reg, range) } +/// How many vregs an allocation keeps in a physical register. +/// +/// The rest are spilled to the frame or recomputed on use, both of which cost +/// instructions the register form does not. +fn registers_held(allocation: &IndexMap>>) -> usize { + allocation + .iter() + .filter(|alloc| matches!(alloc, Some(Allocation::Register(_)))) + .count() +} + +/// Whether a reuse pass's result may replace the baseline pass's. +/// +/// The reuse pass (see [`linear_scan_impl_with_remat`]) re-runs assignment +/// allowing call-free intervals to occupy callee-saved registers the baseline +/// pass already committed to the prologue. That is a codegen-quality trade with +/// no intended effect on frame cost, so it is taken only when it costs nothing: +/// +/// * **No new save.** Occupying an already-saved register denies it to a later +/// call-crossing interval, which may then reach for a *fresh* callee-saved +/// register — a save the baseline did not pay. Requiring the reuse pass's +/// save set to be contained in the baseline's rejects exactly that, and with +/// it any risk of undoing the saves RUE-1146 removed. Containment is the +/// right test rather than a count: a same-size but different save set would +/// mean the reuse pass forced a register the baseline never touched. +/// * **No value displaced from a register.** Denying the caller-saved class to +/// an interval that could have used it can raise pressure enough that +/// something no longer fits. Counting the vregs still in registers catches +/// that whether the loser ends up spilled or rematerialized. +/// * **No spill in place of a rematerialization.** The register count alone +/// would let the two trade places, and a spill costs the frame traffic a +/// recompute does not. +fn accept_reuse_pass( + baseline: &( + IndexMap>>, + u32, + Vec, + RegAllocDebugInfo, + ), + reuse: &( + IndexMap>>, + u32, + Vec, + RegAllocDebugInfo, + ), +) -> bool { + let (baseline_allocation, baseline_spills, baseline_saved, _) = baseline; + let (reuse_allocation, reuse_spills, reuse_saved, _) = reuse; + reuse_saved.iter().all(|reg| baseline_saved.contains(reg)) + && registers_held(reuse_allocation) >= registers_held(baseline_allocation) + && reuse_spills <= baseline_spills +} + /// Internal implementation of linear scan register allocation. /// /// This is the shared implementation used by both [`linear_scan`] and -/// [`linear_scan_with_debug`]. When `collect_debug` is `false` (the normal -/// compilation path, where callers discard the debug info) the O(V²) -/// interference-graph construction is skipped — it feeds only `--emit regalloc` -/// output and building it on every compile made allocation quadratic in the -/// number of virtual registers (e.g. large array literals) (RUE-302). +/// [`linear_scan_with_debug`]. See [`linear_scan_impl_with_remat`] for the +/// two-pass structure, which is identical here. fn linear_scan_impl( vreg_count: u32, liveness: &LivenessInfo, @@ -1796,6 +1880,71 @@ fn linear_scan_impl( u32, Vec, RegAllocDebugInfo, +) { + let baseline = scan_intervals( + vreg_count, + liveness, + classes, + existing_locals, + collect_debug, + cost_model, + loop_info, + &[], + ); + let (_, _, baseline_saved, _) = &baseline; + // The reuse pass can only differ where a compact callee-saved register is + // already in the save set and there is a caller-saved register to prefer it + // over. Otherwise skip it entirely, so neither a push-free function nor a + // fixed-width target pays for a second scan. + if classes.caller_saved.is_empty() + || !classes + .compact_callee_saved + .iter() + .any(|reg| baseline_saved.contains(reg)) + { + return baseline; + } + let reuse = scan_intervals( + vreg_count, + liveness, + classes, + existing_locals, + collect_debug, + cost_model, + loop_info, + baseline_saved, + ); + if accept_reuse_pass(&baseline, &reuse) { + reuse + } else { + baseline + } +} + +/// One linear-scan pass over the intervals. +/// +/// When `collect_debug` is `false` (the normal compilation path, where callers +/// discard the debug info) the O(V²) interference-graph construction is skipped +/// — it feeds only `--emit regalloc` output and building it on every compile +/// made allocation quadratic in the number of virtual registers (e.g. large +/// array literals) (RUE-302). +/// +/// `sunk` names callee-saved registers whose prologue save is already paid for; +/// see [`pick_free_register`]. +fn scan_intervals( + vreg_count: u32, + liveness: &LivenessInfo, + classes: RegisterClasses<'_, Reg>, + existing_locals: u32, + collect_debug: bool, + cost_model: &CostModel, + loop_info: &LoopInfo, + sunk: &[Reg], +) -> ( + IndexMap>>, + u32, + Vec, + RegAllocDebugInfo, ) { let vreg_count_usize = vreg_count as usize; @@ -1855,8 +2004,9 @@ fn linear_scan_impl( // Find registers currently in use let used_regs: HashSet = active.iter().map(|&(_, reg, _)| reg).collect(); - // Try to find a free register, caller-saved class first - let allocated_reg = pick_free_register(classes, &clobbers, &used_regs, &range); + // Try to find a free register: a sunk compact one, else caller-saved, + // else a fresh callee-saved one. + let allocated_reg = pick_free_register(classes, &clobbers, &used_regs, sunk, &range); if let Some(reg) = allocated_reg { // Assign this register @@ -1945,10 +2095,95 @@ fn linear_scan_impl( /// Internal implementation of linear scan with rematerialization support. /// +/// This is the production allocation entry point for both backends. +/// +/// Assignment runs in two passes. The first is the RUE-1146 policy on its own: +/// every interval that can live in a caller-saved register does, so a function +/// whose values all fit there saves nothing in its prologue. If that pass ends +/// up committing no callee-saved register — the push-free case RUE-1146 exists +/// to produce — the answer is already final and the second pass is skipped +/// outright, so the guarantee is structural rather than measured. +/// +/// Otherwise a second pass re-runs assignment knowing which callee-saved +/// registers the function pays for regardless. Those are then preferred over a +/// caller-saved register for a call-free interval, because their cost is +/// already sunk while the caller-saved register's addressing cost is not: on +/// x86-64 the one caller-saved candidate is `r11`, whose byte and dword forms +/// each pay a REX prefix that `rbx` does not (RUE-1227). Knowing the final save +/// set is what the second pass buys — a single pass cannot, since intervals are +/// processed in start order and a later interval can force a register into the +/// save set after an earlier one has already chosen against it. +/// +/// The second pass's result is taken only if [`accept_reuse_pass`] agrees it +/// costs no additional save and no additional spill; otherwise the first pass's +/// allocation stands. Both passes produce a complete, independently valid +/// allocation, so choosing between them needs no repair step. +fn linear_scan_impl_with_remat( + vreg_count: u32, + liveness: &LivenessInfo, + classes: RegisterClasses<'_, Reg>, + existing_locals: u32, + collect_debug: bool, + cost_model: &CostModel, + loop_info: &LoopInfo, + vreg_info: &IndexMap, +) -> ( + IndexMap>>, + u32, + Vec, + RegAllocDebugInfo, +) { + let baseline = scan_intervals_with_remat( + vreg_count, + liveness, + classes, + existing_locals, + collect_debug, + cost_model, + loop_info, + vreg_info, + &[], + ); + let (_, _, baseline_saved, _) = &baseline; + // The reuse pass can only differ where a compact callee-saved register is + // already in the save set and there is a caller-saved register to prefer it + // over. Otherwise skip it entirely, so neither a push-free function nor a + // fixed-width target pays for a second scan. + if classes.caller_saved.is_empty() + || !classes + .compact_callee_saved + .iter() + .any(|reg| baseline_saved.contains(reg)) + { + return baseline; + } + let reuse = scan_intervals_with_remat( + vreg_count, + liveness, + classes, + existing_locals, + collect_debug, + cost_model, + loop_info, + vreg_info, + baseline_saved, + ); + if accept_reuse_pass(&baseline, &reuse) { + reuse + } else { + baseline + } +} + +/// One linear-scan pass with rematerialization support. +/// /// When a vreg needs to be spilled but has rematerialization info, it is marked /// for rematerialization instead of being allocated a stack slot. This avoids /// memory traffic for values that can be cheaply recomputed (constants, etc.). -fn linear_scan_impl_with_remat( +/// +/// `sunk` names callee-saved registers whose prologue save is already paid for; +/// see [`pick_free_register`]. +fn scan_intervals_with_remat( vreg_count: u32, liveness: &LivenessInfo, classes: RegisterClasses<'_, Reg>, @@ -1957,6 +2192,7 @@ fn linear_scan_impl_with_remat( cost_model: &CostModel, loop_info: &LoopInfo, vreg_info: &IndexMap, + sunk: &[Reg], ) -> ( IndexMap>>, u32, @@ -2025,8 +2261,9 @@ fn linear_scan_impl_with_remat( // Find registers currently in use let used_regs: HashSet = active.iter().map(|&(_, reg, _)| reg).collect(); - // Try to find a free register, caller-saved class first - let allocated_reg = pick_free_register(classes, &clobbers, &used_regs, &range); + // Try to find a free register: a sunk compact one, else caller-saved, + // else a fresh callee-saved one. + let allocated_reg = pick_free_register(classes, &clobbers, &used_regs, sunk, &range); if let Some(reg) = allocated_reg { // Assign this register @@ -2290,6 +2527,7 @@ mod tests { let classes = RegisterClasses { caller_saved: &[TestReg(9), TestReg(8)], callee_saved: &[TestReg(0)], + compact_callee_saved: &[], }; let (allocation, num_spills, used_callee_saved, _) = linear_scan_impl( 2, @@ -2348,6 +2586,7 @@ mod tests { let classes = RegisterClasses { caller_saved: &[TestReg(9), TestReg(8)], callee_saved: &[TestReg(0)], + compact_callee_saved: &[], }; let (allocation, num_spills, used_callee_saved, _) = linear_scan_impl( 2, @@ -2377,6 +2616,134 @@ mod tests { ); } + #[test] + fn a_call_free_interval_reuses_a_compact_register_the_prologue_already_saves() { + // Same shape as `caller_saved_is_preferred_...`, but now the callee- + // saved register is the compact one. v0 spans the clobber and forces + // the save; v1 does not, and would take the caller-saved register on + // its own. Because v0's save is paid either way and TestReg(0) encodes + // better, the second pass gives v1 the callee-saved register instead + // (RUE-1227) — at no cost, since the prologue is unchanged. + let liveness = make_liveness_with_clobbers( + vec![(0, 0, 2), (1, 3, 4)], + vec![(2, TestReg(9)), (2, TestReg(8))], + ); + let classes = RegisterClasses { + caller_saved: &[TestReg(9), TestReg(8)], + callee_saved: &[TestReg(0)], + compact_callee_saved: &[TestReg(0)], + }; + let (allocation, num_spills, used_callee_saved, _) = linear_scan_impl( + 2, + &liveness, + classes, + 0, + false, + &CostModel::default(), + &LoopInfo::no_loops(liveness.live_at.len()), + ); + + assert_eq!(num_spills, 0); + assert_eq!( + allocation[VReg::new(0)], + Some(Allocation::Register(TestReg(0))) + ); + assert_eq!( + allocation[VReg::new(1)], + Some(Allocation::Register(TestReg(0))), + "a call-free interval should reuse the freed compact register, not \ + reach for the caller-saved class" + ); + assert_eq!(used_callee_saved, vec![TestReg(0)], "no new save"); + } + + #[test] + fn a_call_free_function_never_reuses_a_callee_saved_register() { + // The RUE-1146 invariant the tiebreak must not undo: nothing here + // spans the clobber, so the first pass commits no callee-saved + // register at all, there is no sunk cost to reuse, and the second pass + // is skipped outright. The prologue stays empty. + let liveness = + make_liveness_with_clobbers(vec![(0, 0, 1), (1, 3, 4)], vec![(2, TestReg(9))]); + let classes = RegisterClasses { + caller_saved: &[TestReg(9)], + callee_saved: &[TestReg(0)], + compact_callee_saved: &[TestReg(0)], + }; + let (allocation, num_spills, used_callee_saved, _) = linear_scan_impl( + 2, + &liveness, + classes, + 0, + false, + &CostModel::default(), + &LoopInfo::no_loops(liveness.live_at.len()), + ); + + assert_eq!(num_spills, 0); + assert_eq!( + allocation[VReg::new(0)], + Some(Allocation::Register(TestReg(9))) + ); + assert_eq!( + allocation[VReg::new(1)], + Some(Allocation::Register(TestReg(9))) + ); + assert!( + used_callee_saved.is_empty(), + "a function whose values all fit caller-saved must stay push-free" + ); + } + + #[test] + fn the_reuse_pass_is_rejected_when_it_would_force_a_second_save() { + // The case the acceptance check exists for. v0 spans the first clobber + // and takes the compact register, then expires; v1 is call-free and + // would happily reuse that register; but v2 spans the second clobber + // and overlaps v1, so handing v1 the compact register pushes v2 onto a + // *fresh* callee-saved register the first pass never touched. That is a + // new prologue save, so the reuse pass is discarded and the first + // pass's allocation stands. + // + // This is exactly why the tiebreak cannot be a one-pass rule that + // reuses whatever is in the save set so far: at v1 the allocator has + // not yet seen v2. + let liveness = make_liveness_with_clobbers( + vec![(0, 0, 2), (1, 3, 5), (2, 4, 9)], + vec![(2, TestReg(9)), (7, TestReg(9))], + ); + let classes = RegisterClasses { + caller_saved: &[TestReg(9)], + callee_saved: &[TestReg(0), TestReg(1)], + compact_callee_saved: &[TestReg(0)], + }; + let (allocation, num_spills, used_callee_saved, _) = linear_scan_impl( + 3, + &liveness, + classes, + 0, + false, + &CostModel::default(), + &LoopInfo::no_loops(liveness.live_at.len()), + ); + + assert_eq!(num_spills, 0); + assert_eq!( + allocation[VReg::new(1)], + Some(Allocation::Register(TestReg(9))), + "the call-free interval keeps the caller-saved register" + ); + assert_eq!( + allocation[VReg::new(2)], + Some(Allocation::Register(TestReg(0))) + ); + assert_eq!( + used_callee_saved, + vec![TestReg(0)], + "the tiebreak must never enlarge the save set" + ); + } + #[test] fn eviction_never_hands_a_clobbered_caller_saved_register_to_a_spanning_interval() { // Only a caller-saved register exists, and every interval spans the @@ -2388,6 +2755,7 @@ mod tests { let classes = RegisterClasses { caller_saved: &[TestReg(9)], callee_saved: &[], + compact_callee_saved: &[], }; let (allocation, num_spills, used_callee_saved, _) = linear_scan_impl( 3, diff --git a/crates/rue-codegen/src/x86_64/regalloc.rs b/crates/rue-codegen/src/x86_64/regalloc.rs index 75426e4d8..159436509 100644 --- a/crates/rue-codegen/src/x86_64/regalloc.rs +++ b/crates/rue-codegen/src/x86_64/regalloc.rs @@ -39,8 +39,35 @@ const CALLER_SAVED_REGS: &[Reg] = &[Reg::R11]; /// /// Each one used obliges the prologue to save it and the epilogue to restore /// it, so allocation reaches for these only after the caller-saved class above -/// is exhausted or ineligible. -const CALLEE_SAVED_REGS: &[Reg] = &[Reg::R12, Reg::R13, Reg::R14, Reg::R15, Reg::Rbx]; +/// is exhausted or ineligible — with the one exception in +/// [`COMPACT_CALLEE_SAVED_REGS`], which costs no additional save. +/// +/// The order is by encoding cost, so a function that needs fewer of them than +/// there are gets the cheapest ones (RUE-1227). +/// +/// `rbx` leads: it is the only legacy register here, so its byte and dword +/// forms encode without the REX prefix `r12`-`r15` always need, and `push rbx` +/// / `pop rbx` are a byte shorter than the extended forms. +/// +/// `r12` trails: its low three bits are `rsp`'s, so *every* memory operand +/// based on it needs a SIB byte that no other register here does. That costs a +/// byte per access, and allocation hands long-lived pointers to callee-saved +/// registers precisely because they are used a lot — an aggregate base held in +/// `r12` paid for itself a hundred times over in `examples/life`. +/// +/// `r13`-`r15` sit between, in numeric order; they encode identically to each +/// other for every form allocation produces. +const CALLEE_SAVED_REGS: &[Reg] = &[Reg::Rbx, Reg::R13, Reg::R14, Reg::R15, Reg::R12]; + +/// Callee-saved registers that encode at least as compactly as any caller-saved +/// one, and so are worth preferring over [`CALLER_SAVED_REGS`] for a call-free +/// interval once their prologue save is already paid for (RUE-1227). +/// +/// `rbx` is the whole set: `r11` and `r12`-`r15` are all extended registers +/// that pay the same REX prefix as each other, so trading `r11` for one of them +/// would give up a register and buy nothing. See +/// [`RegisterClasses::compact_callee_saved`]. +const COMPACT_CALLEE_SAVED_REGS: &[Reg] = &[Reg::Rbx]; /// Every allocatable register, in preference order: caller-saved first. /// @@ -50,11 +77,11 @@ const CALLEE_SAVED_REGS: &[Reg] = &[Reg::R12, Reg::R13, Reg::R14, Reg::R15, Reg: /// has a register available, values are spilled to the stack. const ALLOCATABLE_REGS: &[Reg] = &[ Reg::R11, // Caller-saved - Reg::R12, // Callee-saved + Reg::Rbx, // Callee-saved Reg::R13, // Callee-saved Reg::R14, // Callee-saved Reg::R15, // Callee-saved - Reg::Rbx, // Callee-saved + Reg::R12, // Callee-saved ]; // No allocatable register may carry a reserved role, and the flattened list @@ -83,6 +110,26 @@ const _: () = { ); index += 1; } + // The compact set is a preference among callee-saved registers, so a + // register outside that class must never appear in it — offering one would + // hand a call-crossing interval a register no prologue saves. + let mut index = 0; + while index < COMPACT_CALLEE_SAVED_REGS.len() { + let mut found = false; + let mut probe = 0; + while probe < CALLEE_SAVED_REGS.len() { + if CALLEE_SAVED_REGS[probe] as u8 == COMPACT_CALLEE_SAVED_REGS[index] as u8 { + found = true; + } + probe += 1; + } + assert!( + found, + "a compact register must be callee-saved: the RUE-1227 preference \ + only ever reuses a register the prologue already saves" + ); + index += 1; + } }; /// Zero-sized adapter for target-specific analysis and instruction rewriting. @@ -1375,6 +1422,7 @@ impl RegAllocBackend for X86Backend { RegisterClasses { caller_saved: CALLER_SAVED_REGS, callee_saved: CALLEE_SAVED_REGS, + compact_callee_saved: COMPACT_CALLEE_SAVED_REGS, } } @@ -1417,8 +1465,8 @@ impl RegAllocBackend for X86Backend { mod tests { use super::liveness; use super::{ - ALLOCATABLE_REGS, CALLEE_SAVED_REGS, CALLER_SAVED_REGS, Operand, Reg, RegAlloc, VReg, - X86Inst, X86Mir, + ALLOCATABLE_REGS, CALLEE_SAVED_REGS, CALLER_SAVED_REGS, COMPACT_CALLEE_SAVED_REGS, Operand, + Reg, RegAlloc, VReg, X86Inst, X86Mir, }; use crate::regalloc::{Allocation, RematerializeOp}; @@ -1985,6 +2033,83 @@ mod tests { ); } + #[test] + fn the_callee_saved_order_leads_with_rbx_and_trails_with_r12() { + // The order is an encoding-cost claim, not an arbitrary listing, and + // both ends of it are load-bearing (RUE-1227): `rbx` needs no REX + // prefix for byte and dword forms, and `r12` needs a SIB byte for every + // memory operand based on it. A function that uses fewer callee-saved + // registers than exist must get the cheap end first. + assert_eq!(CALLEE_SAVED_REGS.first(), Some(&Reg::Rbx)); + assert_eq!(CALLEE_SAVED_REGS.last(), Some(&Reg::R12)); + assert_eq!( + COMPACT_CALLEE_SAVED_REGS, + &[Reg::Rbx], + "r11 and r12-r15 are all extended registers that pay the same REX \ + prefix, so only rbx is worth taking back from the caller-saved class" + ); + } + + #[test] + fn a_call_free_value_prefers_rbx_over_r11_once_rbx_is_saved() { + // A value defined before a call and used after it forces `rbx` into the + // prologue. A second, call-free value then costs nothing extra to put + // in `rbx` as well once the first has died, and encodes better there + // than in `r11` (RUE-1227). The save set must not grow to pay for it. + let mut mir = X86Mir::new(); + let symbol = mir.intern_symbol("callee"); + let across = mir.alloc_vreg(); + let after = mir.alloc_vreg(); + + mir.push(X86Inst::MovRM { + dst: Operand::Virtual(across), + base: Reg::Rsi, + offset: 0, + }); + mir.push(X86Inst::call(symbol)); + mir.push(X86Inst::MovRR { + dst: Operand::Physical(Reg::Rdi), + src: Operand::Virtual(across), + }); + mir.push(X86Inst::MovRM { + dst: Operand::Virtual(after), + base: Reg::Rsi, + offset: 8, + }); + mir.push(X86Inst::MovRR { + dst: Operand::Physical(Reg::Rdi), + src: Operand::Virtual(after), + }); + + let (mir, num_spills, used_callee_saved) = + RegAlloc::new(mir, 0).allocate_with_spills().unwrap(); + + assert_eq!(num_spills, 0); + assert_eq!( + used_callee_saved, + vec![Reg::Rbx], + "the call-crossing value alone decides the prologue" + ); + + let loads: Vec = mir + .instructions() + .iter() + .filter_map(|inst| match inst { + X86Inst::MovRM { + dst: Operand::Physical(reg), + base: Reg::Rsi, + .. + } => Some(*reg), + _ => None, + }) + .collect(); + assert_eq!( + loads, + vec![Reg::Rbx, Reg::Rbx], + "the call-free value should reuse the already-saved rbx, not take r11" + ); + } + #[test] fn cross_call_values_never_take_a_caller_saved_register() { // Every value here is defined before a call and used after it, so none