From 34933d018b08a383e0c3bbaba8eabb2c7bca3001 Mon Sep 17 00:00:00 2001 From: Jonas Kruckenberg Date: Mon, 13 Jan 2025 11:16:54 +0100 Subject: [PATCH 1/7] feat: make `MachineEnv`s static and remove `OnceLock`s --- cranelift/codegen/src/isa/aarch64/abi.rs | 130 +++++++++++--- .../codegen/src/isa/aarch64/inst/regs.rs | 10 +- .../codegen/src/isa/pulley_shared/abi.rs | 61 +++---- cranelift/codegen/src/isa/riscv64/abi.rs | 169 ++++++++++++------ cranelift/codegen/src/isa/s390x/abi.rs | 58 +++--- cranelift/codegen/src/isa/x64/abi.rs | 115 +++++++++--- cranelift/codegen/src/isa/x64/inst/regs.rs | 70 ++++---- cranelift/codegen/src/machinst/reg.rs | 41 ++++- 8 files changed, 442 insertions(+), 212 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 3008a42f687d..21c871573e0f 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -16,8 +16,6 @@ use alloc::boxed::Box; use alloc::vec::Vec; use regalloc2::{MachineEnv, PReg, PRegSet}; use smallvec::{smallvec, SmallVec}; -use std::borrow::ToOwned; -use std::sync::OnceLock; // We use a generic implementation that factors out AArch64 and x64 ABI commonalities, because // these ABIs are very similar. @@ -1115,11 +1113,9 @@ impl ABIMachineSpec for AArch64MachineDeps { fn get_machine_env(flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { if flags.enable_pinned_reg() { - static MACHINE_ENV: OnceLock = OnceLock::new(); - MACHINE_ENV.get_or_init(|| create_reg_env(true)) + &PINNED_MACHINE_ENV } else { - static MACHINE_ENV: OnceLock = OnceLock::new(); - MACHINE_ENV.get_or_init(|| create_reg_env(false)) + &DEFAULT_MACHINE_ENV } } @@ -1440,14 +1436,14 @@ const fn default_aapcs_clobbers() -> PRegSet { const DEFAULT_AAPCS_CLOBBERS: PRegSet = default_aapcs_clobbers(); -fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { - fn preg(r: Reg) -> PReg { - r.to_real_reg().unwrap().into() - } +const fn preg(r: Reg) -> PReg { + r.to_physical_reg().unwrap() +} - let mut env = MachineEnv { +static PINNED_MACHINE_ENV: MachineEnv = { + MachineEnv { preferred_regs_by_class: [ - vec![ + &[ preg(xreg(0)), preg(xreg(1)), preg(xreg(2)), @@ -1471,7 +1467,7 @@ fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { // x21 is the pinned register (if enabled) and not allocatable if so. // x29 is FP, x30 is LR, x31 is SP/ZR. ], - vec![ + &[ preg(vreg(0)), preg(vreg(1)), preg(vreg(2)), @@ -1499,13 +1495,12 @@ fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { preg(vreg(31)), ], // Vector Regclass is unused - vec![], + &[], ], non_preferred_regs_by_class: [ - vec![ + &[ preg(xreg(19)), preg(xreg(20)), - // x21 is pinned reg if enabled; we add to this list below if not. preg(xreg(22)), preg(xreg(23)), preg(xreg(24)), @@ -1514,7 +1509,7 @@ fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { preg(xreg(27)), preg(xreg(28)), ], - vec![ + &[ preg(vreg(8)), preg(vreg(9)), preg(vreg(10)), @@ -1525,16 +1520,99 @@ fn create_reg_env(enable_pinned_reg: bool) -> MachineEnv { preg(vreg(15)), ], // Vector Regclass is unused - vec![], + &[], ], - fixed_stack_slots: vec![], + fixed_stack_slots: &[], scratch_by_class: [None, None, None], - }; - - if !enable_pinned_reg { - debug_assert_eq!(PINNED_REG, 21); // We assumed this above in hardcoded reg list. - env.non_preferred_regs_by_class[0].push(preg(xreg(PINNED_REG))); } +}; - env -} +static DEFAULT_MACHINE_ENV: MachineEnv = { + debug_assert!(PINNED_REG == 21); // We assumed this below in hardcoded reg list. + + MachineEnv { + preferred_regs_by_class: [ + &[ + preg(xreg(0)), + preg(xreg(1)), + preg(xreg(2)), + preg(xreg(3)), + preg(xreg(4)), + preg(xreg(5)), + preg(xreg(6)), + preg(xreg(7)), + preg(xreg(8)), + preg(xreg(9)), + preg(xreg(10)), + preg(xreg(11)), + preg(xreg(12)), + preg(xreg(13)), + preg(xreg(14)), + preg(xreg(15)), + // x16 and x17 are spilltmp and tmp2 (see above). + // x18 could be used by the platform to carry inter-procedural state; + // conservatively assume so and make it not allocatable. + // x19-28 are callee-saved and so not preferred. + // x21 is the pinned register (if enabled) and not allocatable if so. + // x29 is FP, x30 is LR, x31 is SP/ZR. + ], + &[ + preg(vreg(0)), + preg(vreg(1)), + preg(vreg(2)), + preg(vreg(3)), + preg(vreg(4)), + preg(vreg(5)), + preg(vreg(6)), + preg(vreg(7)), + // v8-15 are callee-saved and so not preferred. + preg(vreg(16)), + preg(vreg(17)), + preg(vreg(18)), + preg(vreg(19)), + preg(vreg(20)), + preg(vreg(21)), + preg(vreg(22)), + preg(vreg(23)), + preg(vreg(24)), + preg(vreg(25)), + preg(vreg(26)), + preg(vreg(27)), + preg(vreg(28)), + preg(vreg(29)), + preg(vreg(30)), + preg(vreg(31)), + ], + // Vector Regclass is unused + &[], + ], + non_preferred_regs_by_class: [ + &[ + preg(xreg(19)), + preg(xreg(20)), + preg(xreg(22)), + preg(xreg(23)), + preg(xreg(24)), + preg(xreg(25)), + preg(xreg(26)), + preg(xreg(27)), + preg(xreg(28)), + preg(xreg(PINNED_REG)), + ], + &[ + preg(vreg(8)), + preg(vreg(9)), + preg(vreg(10)), + preg(vreg(11)), + preg(vreg(12)), + preg(vreg(13)), + preg(vreg(14)), + preg(vreg(15)), + ], + // Vector Regclass is unused + &[], + ], + fixed_stack_slots: &[], + scratch_by_class: [None, None, None], + } +}; diff --git a/cranelift/codegen/src/isa/aarch64/inst/regs.rs b/cranelift/codegen/src/isa/aarch64/inst/regs.rs index fd1abb7fffb4..262e2c5fdd0a 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/regs.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/regs.rs @@ -20,8 +20,8 @@ pub const PINNED_REG: u8 = 21; /// Get a reference to an X-register (integer register). Do not use /// this for xsp / xzr; we have two special registers for those. -pub fn xreg(num: u8) -> Reg { - Reg::from(xreg_preg(num)) +pub const fn xreg(num: u8) -> Reg { + Reg::from_preg(xreg_preg(num)) } /// Get the given X-register as a PReg. @@ -36,8 +36,8 @@ pub fn writable_xreg(num: u8) -> Writable { } /// Get a reference to a V-register (vector/FP register). -pub fn vreg(num: u8) -> Reg { - Reg::from(vreg_preg(num)) +pub const fn vreg(num: u8) -> Reg { + Reg::from_preg(vreg_preg(num)) } /// Get the given V-register as a PReg. @@ -101,7 +101,7 @@ pub fn writable_link_reg() -> Writable { } /// Get a reference to the frame pointer (x29). -pub fn fp_reg() -> Reg { +pub const fn fp_reg() -> Reg { xreg(29) } diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs index 6efbd3927b2b..f13adb751f64 100644 --- a/cranelift/codegen/src/isa/pulley_shared/abi.rs +++ b/cranelift/codegen/src/isa/pulley_shared/abi.rs @@ -525,8 +525,7 @@ where } fn get_machine_env(_flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { - static MACHINE_ENV: OnceLock = OnceLock::new(); - MACHINE_ENV.get_or_init(create_reg_environment) + &DEFAULT_MACHINE_ENV } fn get_regs_clobbered_by_call(_call_conv_of_callee: isa::CallConv) -> PRegSet { @@ -913,31 +912,33 @@ const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() .with(pv_reg(30)) .with(pv_reg(31)); -fn create_reg_environment() -> MachineEnv { - // Prefer caller-saved registers over callee-saved registers, because that - // way we don't need to emit code to save and restore them if we don't - // mutate them. - - let preferred_regs_by_class: [Vec; 3] = { - let x_registers: Vec = (0..16).map(|x| px_reg(x)).collect(); - let f_registers: Vec = (0..16).map(|x| pf_reg(x)).collect(); - let v_registers: Vec = (0..32).map(|x| pv_reg(x)).collect(); - [x_registers, f_registers, v_registers] - }; - - let non_preferred_regs_by_class: [Vec; 3] = { - let x_registers: Vec = (16..XReg::SPECIAL_START) - .map(|x| px_reg(x.into())) - .collect(); - let f_registers: Vec = (16..32).map(|x| pf_reg(x)).collect(); - let v_registers: Vec = vec![]; - [x_registers, f_registers, v_registers] - }; - - MachineEnv { - preferred_regs_by_class, - non_preferred_regs_by_class, - fixed_stack_slots: vec![], - scratch_by_class: [None, None, None], - } -} +static DEFAULT_MACHINE_ENV: MachineEnv = { todo!() }; + +// fn create_reg_environment() -> MachineEnv { +// // Prefer caller-saved registers over callee-saved registers, because that +// // way we don't need to emit code to save and restore them if we don't +// // mutate them. +// +// let preferred_regs_by_class: [Vec; 3] = { +// let x_registers: Vec = (0..16).map(|x| px_reg(x)).collect(); +// let f_registers: Vec = (0..16).map(|x| pf_reg(x)).collect(); +// let v_registers: Vec = (0..32).map(|x| pv_reg(x)).collect(); +// [x_registers, f_registers, v_registers] +// }; +// +// let non_preferred_regs_by_class: [Vec; 3] = { +// let x_registers: Vec = (16..XReg::SPECIAL_START) +// .map(|x| px_reg(x.into())) +// .collect(); +// let f_registers: Vec = (16..32).map(|x| pf_reg(x)).collect(); +// let v_registers: Vec = vec![]; +// [x_registers, f_registers, v_registers] +// }; +// +// MachineEnv { +// preferred_regs_by_class, +// non_preferred_regs_by_class, +// fixed_stack_slots: vec![], +// scratch_by_class: [None, None, None], +// } +// } diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs index 3bde5ea9bb17..91d9001cf9c0 100644 --- a/cranelift/codegen/src/isa/riscv64/abi.rs +++ b/cranelift/codegen/src/isa/riscv64/abi.rs @@ -17,11 +17,8 @@ use crate::settings; use crate::CodegenResult; use alloc::boxed::Box; use alloc::vec::Vec; -use regalloc2::{MachineEnv, PReg, PRegSet}; - +use regalloc2::{MachineEnv, PRegSet}; use smallvec::{smallvec, SmallVec}; -use std::borrow::ToOwned; -use std::sync::OnceLock; /// Support for the Riscv64 ABI from the callee side (within a function body). pub(crate) type Riscv64Callee = Callee; @@ -633,8 +630,7 @@ impl ABIMachineSpec for Riscv64MachineDeps { } fn get_machine_env(_flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { - static MACHINE_ENV: OnceLock = OnceLock::new(); - MACHINE_ENV.get_or_init(create_reg_environment) + &DEFAULT_MACHINE_ENV } fn get_regs_clobbered_by_call(_call_conv_of_callee: isa::CallConv) -> PRegSet { @@ -894,7 +890,7 @@ const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() .with(pv_reg(30)) .with(pv_reg(31)); -fn create_reg_environment() -> MachineEnv { +static DEFAULT_MACHINE_ENV: MachineEnv = { // Some C Extension instructions can only use a subset of the registers. // x8 - x15, f8 - f15, v8 - v15 so we should prefer to use those since // they allow us to emit C instructions more often. @@ -905,53 +901,124 @@ fn create_reg_environment() -> MachineEnv { // 3. Compressible Callee Saved registers. // 4. Non-Compressible Callee Saved registers. - let preferred_regs_by_class: [Vec; 3] = { - let x_registers: Vec = (10..=15).map(px_reg).collect(); - let f_registers: Vec = (10..=15).map(pf_reg).collect(); - let v_registers: Vec = (8..=15).map(pv_reg).collect(); - - [x_registers, f_registers, v_registers] - }; - - let non_preferred_regs_by_class: [Vec; 3] = { - // x0 - x4 are special registers, so we don't want to use them. - // Omit x30 and x31 since they are the spilltmp registers. - - // Start with the Non-Compressible Caller Saved registers. - let x_registers: Vec = (5..=7) - .chain(16..=17) - .chain(28..=29) - // The first Callee Saved register is x9 since its Compressible - // Omit x8 since it's the frame pointer. - .chain(9..=9) - // The rest of the Callee Saved registers are Non-Compressible - .chain(18..=27) - .map(px_reg) - .collect(); - - // Prefer Caller Saved registers. - let f_registers: Vec = (0..=7) - .chain(16..=17) - .chain(28..=31) - // Once those are exhausted, we should prefer f8 and f9 since they are - // callee saved, but compressible. - .chain(8..=9) - .chain(18..=27) - .map(pf_reg) - .collect(); - - let v_registers = (0..=7).chain(16..=31).map(pv_reg).collect(); - - [x_registers, f_registers, v_registers] - }; - MachineEnv { - preferred_regs_by_class, - non_preferred_regs_by_class, - fixed_stack_slots: vec![], + preferred_regs_by_class: [ + &[ + px_reg(10), + px_reg(11), + px_reg(12), + px_reg(13), + px_reg(14), + px_reg(15), + ], + &[ + pf_reg(10), + pf_reg(11), + pf_reg(12), + pf_reg(13), + pf_reg(14), + pf_reg(15), + ], + &[ + pv_reg(8), + pv_reg(9), + pv_reg(10), + pv_reg(11), + pv_reg(12), + pv_reg(13), + pv_reg(14), + pv_reg(15), + ], + ], + non_preferred_regs_by_class: [ + // x0 - x4 are special registers, so we don't want to use them. + // Omit x30 and x31 since they are the spilltmp registers. + &[ + // Start with the Non-Compressible Caller Saved registers. + px_reg(6), + px_reg(6), + px_reg(7), + px_reg(16), + px_reg(17), + px_reg(28), + px_reg(29), + // The first Callee Saved register is x9 since its Compressible + // Omit x8 since it's the frame pointer. + px_reg(9), + // The rest of the Callee Saved registers are Non-Compressible + px_reg(18), + px_reg(19), + px_reg(20), + px_reg(21), + px_reg(22), + px_reg(23), + px_reg(24), + px_reg(25), + px_reg(26), + px_reg(27), + ], + &[ + // Prefer Caller Saved registers. + pf_reg(0), + pf_reg(1), + pf_reg(2), + pf_reg(3), + pf_reg(4), + pf_reg(5), + pf_reg(6), + pf_reg(7), + pf_reg(16), + pf_reg(17), + pf_reg(28), + pf_reg(29), + pf_reg(30), + pf_reg(31), + // Once those are exhausted, we should prefer f8 and f9 since they are + // callee saved, but compressible. + pf_reg(8), + pf_reg(9), + pf_reg(18), + pf_reg(19), + pf_reg(20), + pf_reg(21), + pf_reg(22), + pf_reg(23), + pf_reg(24), + pf_reg(25), + pf_reg(26), + pf_reg(27), + ], + &[ + pv_reg(0), + pv_reg(1), + pv_reg(2), + pv_reg(3), + pv_reg(4), + pv_reg(5), + pv_reg(6), + pv_reg(7), + pv_reg(16), + pv_reg(17), + pv_reg(18), + pv_reg(19), + pv_reg(20), + pv_reg(21), + pv_reg(22), + pv_reg(23), + pv_reg(24), + pv_reg(25), + pv_reg(26), + pv_reg(27), + pv_reg(28), + pv_reg(29), + pv_reg(30), + pv_reg(31), + ], + ], scratch_by_class: [None, None, None], + fixed_stack_slots: &[], } -} +}; impl Riscv64MachineDeps { fn gen_probestack_unroll( diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs index 845599cb04a6..dba08ab3401d 100644 --- a/cranelift/codegen/src/isa/s390x/abi.rs +++ b/cranelift/codegen/src/isa/s390x/abi.rs @@ -147,8 +147,6 @@ use crate::CodegenResult; use alloc::vec::Vec; use regalloc2::{MachineEnv, PRegSet}; use smallvec::{smallvec, SmallVec}; -use std::borrow::ToOwned; -use std::sync::OnceLock; // We use a generic implementation that factors out ABI commonalities. @@ -881,14 +879,8 @@ impl ABIMachineSpec for S390xMachineDeps { fn get_machine_env(_flags: &settings::Flags, call_conv: isa::CallConv) -> &MachineEnv { match call_conv { - isa::CallConv::Tail => { - static TAIL_MACHINE_ENV: OnceLock = OnceLock::new(); - TAIL_MACHINE_ENV.get_or_init(tail_create_machine_env) - } - _ => { - static SYSV_MACHINE_ENV: OnceLock = OnceLock::new(); - SYSV_MACHINE_ENV.get_or_init(sysv_create_machine_env) - } + isa::CallConv::Tail => &TAIL_MACHINE_ENV, + _ => &SYSV_MACHINE_ENV, } } @@ -1255,18 +1247,21 @@ const fn tail_clobbers() -> PRegSet { } const TAIL_CLOBBERS: PRegSet = tail_clobbers(); -fn sysv_create_machine_env() -> MachineEnv { +static TAIL_MACHINE_ENV: MachineEnv = { + // Same as the SystemV ABI below, except that %r6 and %r7 are preferred. MachineEnv { preferred_regs_by_class: [ - vec![ + &[ // no r0; can't use for addressing? // no r1; it is our spilltmp. gpr_preg(2), gpr_preg(3), gpr_preg(4), gpr_preg(5), + gpr_preg(6), + gpr_preg(7), ], - vec![ + &[ vr_preg(0), vr_preg(1), vr_preg(2), @@ -1293,12 +1288,10 @@ fn sysv_create_machine_env() -> MachineEnv { vr_preg(31), ], // Vector Regclass is unused - vec![], + &[], ], non_preferred_regs_by_class: [ - vec![ - gpr_preg(6), - gpr_preg(7), + &[ gpr_preg(8), gpr_preg(9), gpr_preg(10), @@ -1308,7 +1301,7 @@ fn sysv_create_machine_env() -> MachineEnv { gpr_preg(14), // no r15; it is the stack pointer. ], - vec![ + &[ vr_preg(8), vr_preg(9), vr_preg(10), @@ -1319,28 +1312,25 @@ fn sysv_create_machine_env() -> MachineEnv { vr_preg(15), ], // Vector Regclass is unused - vec![], + &[], ], - fixed_stack_slots: vec![], scratch_by_class: [None, None, None], + fixed_stack_slots: &[], } -} +}; -fn tail_create_machine_env() -> MachineEnv { - // Same as the SystemV ABI, except that %r6 and %r7 are preferred. +static SYSV_MACHINE_ENV: MachineEnv = { MachineEnv { preferred_regs_by_class: [ - vec![ + &[ // no r0; can't use for addressing? // no r1; it is our spilltmp. gpr_preg(2), gpr_preg(3), gpr_preg(4), gpr_preg(5), - gpr_preg(6), - gpr_preg(7), ], - vec![ + &[ vr_preg(0), vr_preg(1), vr_preg(2), @@ -1367,10 +1357,12 @@ fn tail_create_machine_env() -> MachineEnv { vr_preg(31), ], // Vector Regclass is unused - vec![], + &[], ], non_preferred_regs_by_class: [ - vec![ + &[ + gpr_preg(6), + gpr_preg(7), gpr_preg(8), gpr_preg(9), gpr_preg(10), @@ -1380,7 +1372,7 @@ fn tail_create_machine_env() -> MachineEnv { gpr_preg(14), // no r15; it is the stack pointer. ], - vec![ + &[ vr_preg(8), vr_preg(9), vr_preg(10), @@ -1391,9 +1383,9 @@ fn tail_create_machine_env() -> MachineEnv { vr_preg(15), ], // Vector Regclass is unused - vec![], + &[], ], - fixed_stack_slots: vec![], scratch_by_class: [None, None, None], + fixed_stack_slots: &[], } -} +}; diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 52005855553d..d952e83ba79f 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -15,8 +15,6 @@ use alloc::vec::Vec; use args::*; use regalloc2::{MachineEnv, PReg, PRegSet}; use smallvec::{smallvec, SmallVec}; -use std::borrow::ToOwned; -use std::sync::OnceLock; /// Support for the x64 ABI from the callee side (within a function body). pub(crate) type X64Callee = Callee; @@ -898,11 +896,9 @@ impl ABIMachineSpec for X64ABIMachineSpec { fn get_machine_env(flags: &settings::Flags, _call_conv: isa::CallConv) -> &MachineEnv { if flags.enable_pinned_reg() { - static MACHINE_ENV: OnceLock = OnceLock::new(); - MACHINE_ENV.get_or_init(|| create_reg_env_systemv(true)) + &PINNED_MACHINE_ENV } else { - static MACHINE_ENV: OnceLock = OnceLock::new(); - MACHINE_ENV.get_or_init(|| create_reg_env_systemv(false)) + &DEFAULT_MACHINE_ENV } } @@ -1316,15 +1312,15 @@ const fn all_clobbers() -> PRegSet { .with(regs::fpr_preg(15)) } -fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { - fn preg(r: Reg) -> PReg { - r.to_real_reg().unwrap().into() - } +fn preg(r: Reg) -> PReg { + r.to_physical_reg().unwrap() +} - let mut env = MachineEnv { +static DEFAULT_MACHINE_ENV: MachineEnv = { + MachineEnv { preferred_regs_by_class: [ // Preferred GPRs: caller-saved in the SysV ABI. - vec![ + &[ preg(regs::rsi()), preg(regs::rdi()), preg(regs::rax()), @@ -1337,7 +1333,7 @@ fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { ], // Preferred XMMs: the first 8, which can have smaller encodings // with AVX instructions. - vec![ + &[ preg(regs::xmm0()), preg(regs::xmm1()), preg(regs::xmm2()), @@ -1348,11 +1344,11 @@ fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { preg(regs::xmm7()), ], // The Vector Regclass is unused - vec![], + &[], ], non_preferred_regs_by_class: [ // Non-preferred GPRs: callee-saved in the SysV ABI. - vec![ + &[ preg(regs::rbx()), preg(regs::r12()), preg(regs::r13()), @@ -1360,7 +1356,7 @@ fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { ], // Non-preferred XMMs: the last 8 registers, which can have larger // encodings with AVX instructions. - vec![ + &[ preg(regs::xmm8()), preg(regs::xmm9()), preg(regs::xmm10()), @@ -1371,16 +1367,85 @@ fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { preg(regs::xmm15()), ], // The Vector Regclass is unused - vec![], + &[], ], - fixed_stack_slots: vec![], + fixed_stack_slots: &[], scratch_by_class: [None, None, None], - }; - - debug_assert_eq!(regs::r15(), regs::pinned_reg()); - if !enable_pinned_reg { - env.non_preferred_regs_by_class[0].push(preg(regs::r15())); } +}; - env -} +static PINNED_MACHINE_ENV: MachineEnv = { + debug_assert!(regs::r15() == regs::pinned_reg()); + + MachineEnv { + preferred_regs_by_class: [ + // Preferred GPRs: caller-saved in the SysV ABI. + &[ + preg(regs::rsi()), + preg(regs::rdi()), + preg(regs::rax()), + preg(regs::rcx()), + preg(regs::rdx()), + preg(regs::r8()), + preg(regs::r9()), + preg(regs::r10()), + preg(regs::r11()), + ], + // Preferred XMMs: the first 8, which can have smaller encodings + // with AVX instructions. + &[ + preg(regs::xmm0()), + preg(regs::xmm1()), + preg(regs::xmm2()), + preg(regs::xmm3()), + preg(regs::xmm4()), + preg(regs::xmm5()), + preg(regs::xmm6()), + preg(regs::xmm7()), + ], + // The Vector Regclass is unused + &[], + ], + non_preferred_regs_by_class: [ + // Non-preferred GPRs: callee-saved in the SysV ABI. + &[ + preg(regs::rbx()), + preg(regs::r12()), + preg(regs::r13()), + preg(regs::r14()), + preg(regs::r15()), + ], + // Non-preferred XMMs: the last 8 registers, which can have larger + // encodings with AVX instructions. + &[ + preg(regs::xmm8()), + preg(regs::xmm9()), + preg(regs::xmm10()), + preg(regs::xmm11()), + preg(regs::xmm12()), + preg(regs::xmm13()), + preg(regs::xmm14()), + preg(regs::xmm15()), + ], + // The Vector Regclass is unused + &[], + ], + fixed_stack_slots: &[], + scratch_by_class: [None, None, None], + } +}; + +// fn create_reg_env_systemv(enable_pinned_reg: bool) -> MachineEnv { +// +// +// let mut env = MachineEnv { +// +// }; +// +// +// if !enable_pinned_reg { +// env.non_preferred_regs_by_class[0].push(); +// } +// +// env +// } diff --git a/cranelift/codegen/src/isa/x64/inst/regs.rs b/cranelift/codegen/src/isa/x64/inst/regs.rs index d3a06da70a03..de7257d4d888 100644 --- a/cranelift/codegen/src/isa/x64/inst/regs.rs +++ b/cranelift/codegen/src/isa/x64/inst/regs.rs @@ -31,7 +31,7 @@ pub const ENC_R15: u8 = 15; // Constructors for Regs. -fn gpr(enc: u8) -> Reg { +const fn gpr(enc: u8) -> Reg { let preg = gpr_preg(enc); Reg::from(VReg::new(preg.index(), RegClass::Int)) } @@ -39,65 +39,65 @@ pub(crate) const fn gpr_preg(enc: u8) -> PReg { PReg::new(enc as usize, RegClass::Int) } -pub(crate) fn rsi() -> Reg { +pub(crate) const fn rsi() -> Reg { gpr(ENC_RSI) } -pub(crate) fn rdi() -> Reg { +pub(crate) const fn rdi() -> Reg { gpr(ENC_RDI) } -pub(crate) fn rax() -> Reg { +pub(crate) const fn rax() -> Reg { gpr(ENC_RAX) } -pub(crate) fn rcx() -> Reg { +pub(crate) const fn rcx() -> Reg { gpr(ENC_RCX) } -pub(crate) fn rdx() -> Reg { +pub(crate) const fn rdx() -> Reg { gpr(ENC_RDX) } -pub(crate) fn r8() -> Reg { +pub(crate) const fn r8() -> Reg { gpr(ENC_R8) } -pub(crate) fn r9() -> Reg { +pub(crate) const fn r9() -> Reg { gpr(ENC_R9) } -pub(crate) fn r10() -> Reg { +pub(crate) const fn r10() -> Reg { gpr(ENC_R10) } -pub(crate) fn r11() -> Reg { +pub(crate) const fn r11() -> Reg { gpr(ENC_R11) } -pub(crate) fn r12() -> Reg { +pub(crate) const fn r12() -> Reg { gpr(ENC_R12) } -pub(crate) fn r13() -> Reg { +pub(crate) const fn r13() -> Reg { gpr(ENC_R13) } -pub(crate) fn r14() -> Reg { +pub(crate) const fn r14() -> Reg { gpr(ENC_R14) } -pub(crate) fn rbx() -> Reg { +pub(crate) const fn rbx() -> Reg { gpr(ENC_RBX) } -pub(crate) fn r15() -> Reg { +pub(crate) const fn r15() -> Reg { gpr(ENC_R15) } -pub(crate) fn rsp() -> Reg { +pub(crate) const fn rsp() -> Reg { gpr(ENC_RSP) } -pub(crate) fn rbp() -> Reg { +pub(crate) const fn rbp() -> Reg { gpr(ENC_RBP) } /// The pinned register on this architecture. /// It must be the same as Spidermonkey's HeapReg, as found in this file. /// https://searchfox.org/mozilla-central/source/js/src/jit/x64/Assembler-x64.h#99 -pub(crate) fn pinned_reg() -> Reg { +pub(crate) const fn pinned_reg() -> Reg { r15() } -fn fpr(enc: u8) -> Reg { +const fn fpr(enc: u8) -> Reg { let preg = fpr_preg(enc); Reg::from(VReg::new(preg.index(), RegClass::Float)) } @@ -106,52 +106,52 @@ pub(crate) const fn fpr_preg(enc: u8) -> PReg { PReg::new(enc as usize, RegClass::Float) } -pub(crate) fn xmm0() -> Reg { +pub(crate) const fn xmm0() -> Reg { fpr(0) } -pub(crate) fn xmm1() -> Reg { +pub(crate) const fn xmm1() -> Reg { fpr(1) } -pub(crate) fn xmm2() -> Reg { +pub(crate) const fn xmm2() -> Reg { fpr(2) } -pub(crate) fn xmm3() -> Reg { +pub(crate) const fn xmm3() -> Reg { fpr(3) } -pub(crate) fn xmm4() -> Reg { +pub(crate) const fn xmm4() -> Reg { fpr(4) } -pub(crate) fn xmm5() -> Reg { +pub(crate) const fn xmm5() -> Reg { fpr(5) } -pub(crate) fn xmm6() -> Reg { +pub(crate) const fn xmm6() -> Reg { fpr(6) } -pub(crate) fn xmm7() -> Reg { +pub(crate) const fn xmm7() -> Reg { fpr(7) } -pub(crate) fn xmm8() -> Reg { +pub(crate) const fn xmm8() -> Reg { fpr(8) } -pub(crate) fn xmm9() -> Reg { +pub(crate) const fn xmm9() -> Reg { fpr(9) } -pub(crate) fn xmm10() -> Reg { +pub(crate) const fn xmm10() -> Reg { fpr(10) } -pub(crate) fn xmm11() -> Reg { +pub(crate) const fn xmm11() -> Reg { fpr(11) } -pub(crate) fn xmm12() -> Reg { +pub(crate) const fn xmm12() -> Reg { fpr(12) } -pub(crate) fn xmm13() -> Reg { +pub(crate) const fn xmm13() -> Reg { fpr(13) } -pub(crate) fn xmm14() -> Reg { +pub(crate) const fn xmm14() -> Reg { fpr(14) } -pub(crate) fn xmm15() -> Reg { +pub(crate) const fn xmm15() -> Reg { fpr(15) } diff --git a/cranelift/codegen/src/machinst/reg.rs b/cranelift/codegen/src/machinst/reg.rs index 2670e3ad12c3..ca11d82f9824 100644 --- a/cranelift/codegen/src/machinst/reg.rs +++ b/cranelift/codegen/src/machinst/reg.rs @@ -20,7 +20,7 @@ use serde_derive::{Deserialize, Serialize}; const PINNED_VREGS: usize = 192; /// Convert a `VReg` to its pinned `PReg`, if any. -pub fn pinned_vreg_to_preg(vreg: VReg) -> Option { +pub const fn pinned_vreg_to_preg(vreg: VReg) -> Option { if vreg.vreg() < PINNED_VREGS { Some(PReg::from_index(vreg.vreg())) } else { @@ -48,10 +48,20 @@ pub fn first_user_vreg_index() -> usize { pub struct Reg(VReg); impl Reg { + // FIXME(const-hack) remove in favor of `From` impl + pub(crate) const fn from_preg(preg: PReg) -> Reg { + Reg(RealReg(preg).to_vreg()) + } + /// Get the physical register (`RealReg`), if this register is /// one. - pub fn to_real_reg(self) -> Option { - pinned_vreg_to_preg(self.0).map(RealReg) + pub const fn to_real_reg(self) -> Option { + // FIXME(const-hack) use `.map` + if let Some(preg) = pinned_vreg_to_preg(self.0) { + Some(RealReg(preg)) + } else { + None + } } /// Get the virtual (non-physical) register, if this register is @@ -64,6 +74,16 @@ impl Reg { } } + /// get the physical (non-virtual) register, if this register is one. + pub const fn to_physical_reg(self) -> Option { + // FIXME(const-hack) use try (?) + if let Some(reg) = self.to_real_reg() { + Some(reg.0) + } else { + None + } + } + /// Get the class of this register. pub fn class(self) -> RegClass { self.0.class() @@ -108,7 +128,16 @@ impl AsMut for Reg { #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct RealReg(PReg); +impl RealReg {} + impl RealReg { + /// Get the equivalent virtual register. + pub const fn to_vreg(&self) -> VReg { + // This representation is redundant: the class is implied in the vreg + // index as well as being in the vreg class field. + VReg::new(self.0.index(), self.0.class()) + } + /// Get the class of this register. pub fn class(self) -> RegClass { self.0.class() @@ -230,9 +259,7 @@ impl std::convert::From for regalloc2::VReg { impl std::convert::From for regalloc2::VReg { fn from(reg: RealReg) -> regalloc2::VReg { - // This representation is redundant: the class is implied in the vreg - // index as well as being in the vreg class field. - VReg::new(reg.0.index(), reg.0.class()) + reg.to_vreg() } } @@ -250,7 +277,7 @@ impl std::convert::From for RealReg { impl std::convert::From for Reg { fn from(preg: regalloc2::PReg) -> Reg { - RealReg(preg).into() + Reg::from_preg(preg) } } From 416a5f6f9cd7f3e8e3937756f00cb6fb0e18c7ff Mon Sep 17 00:00:00 2001 From: Jonas Kruckenberg Date: Mon, 13 Jan 2025 11:23:53 +0100 Subject: [PATCH 2/7] Update abi.rs --- .../codegen/src/isa/pulley_shared/abi.rs | 150 ++++++++++++++---- 1 file changed, 120 insertions(+), 30 deletions(-) diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs index f13adb751f64..02dcf836416e 100644 --- a/cranelift/codegen/src/isa/pulley_shared/abi.rs +++ b/cranelift/codegen/src/isa/pulley_shared/abi.rs @@ -912,33 +912,123 @@ const DEFAULT_CLOBBERS: PRegSet = PRegSet::empty() .with(pv_reg(30)) .with(pv_reg(31)); -static DEFAULT_MACHINE_ENV: MachineEnv = { todo!() }; - -// fn create_reg_environment() -> MachineEnv { -// // Prefer caller-saved registers over callee-saved registers, because that -// // way we don't need to emit code to save and restore them if we don't -// // mutate them. -// -// let preferred_regs_by_class: [Vec; 3] = { -// let x_registers: Vec = (0..16).map(|x| px_reg(x)).collect(); -// let f_registers: Vec = (0..16).map(|x| pf_reg(x)).collect(); -// let v_registers: Vec = (0..32).map(|x| pv_reg(x)).collect(); -// [x_registers, f_registers, v_registers] -// }; -// -// let non_preferred_regs_by_class: [Vec; 3] = { -// let x_registers: Vec = (16..XReg::SPECIAL_START) -// .map(|x| px_reg(x.into())) -// .collect(); -// let f_registers: Vec = (16..32).map(|x| pf_reg(x)).collect(); -// let v_registers: Vec = vec![]; -// [x_registers, f_registers, v_registers] -// }; -// -// MachineEnv { -// preferred_regs_by_class, -// non_preferred_regs_by_class, -// fixed_stack_slots: vec![], -// scratch_by_class: [None, None, None], -// } -// } +static DEFAULT_MACHINE_ENV: MachineEnv = { + debug_assert!(XReg::SPECIAL_START == 30); + // Prefer caller-saved registers over callee-saved registers, because that + // way we don't need to emit code to save and restore them if we don't + // mutate them. + + MachineEnv { + preferred_regs_by_class: [ + &[ + px_reg(0), + px_reg(1), + px_reg(2), + px_reg(3), + px_reg(4), + px_reg(5), + px_reg(6), + px_reg(7), + px_reg(8), + px_reg(9), + px_reg(10), + px_reg(11), + px_reg(12), + px_reg(13), + px_reg(14), + px_reg(15), + ], + &[ + pf_reg(0), + pf_reg(1), + pf_reg(2), + pf_reg(3), + pf_reg(4), + pf_reg(5), + pf_reg(6), + pf_reg(7), + pf_reg(8), + pf_reg(9), + pf_reg(10), + pf_reg(11), + pf_reg(12), + pf_reg(13), + pf_reg(14), + pf_reg(15), + ], + &[ + pv_reg(0), + pv_reg(1), + pv_reg(2), + pv_reg(3), + pv_reg(4), + pv_reg(5), + pv_reg(6), + pv_reg(7), + pv_reg(8), + pv_reg(9), + pv_reg(10), + pv_reg(11), + pv_reg(12), + pv_reg(13), + pv_reg(14), + pv_reg(15), + pv_reg(16), + pv_reg(17), + pv_reg(18), + pv_reg(19), + pv_reg(20), + pv_reg(21), + pv_reg(22), + pv_reg(23), + pv_reg(24), + pv_reg(25), + pv_reg(26), + pv_reg(27), + pv_reg(28), + pv_reg(29), + pv_reg(30), + pv_reg(31), + ] + ], + non_preferred_regs_by_class: [ + &[ + px_reg(16), + px_reg(17), + px_reg(18), + px_reg(19), + px_reg(20), + px_reg(21), + px_reg(22), + px_reg(23), + px_reg(24), + px_reg(25), + px_reg(26), + px_reg(27), + px_reg(28), + px_reg(29), + ], + &[ + pf_reg(16), + pf_reg(17), + pf_reg(18), + pf_reg(19), + pf_reg(20), + pf_reg(21), + pf_reg(22), + pf_reg(23), + pf_reg(24), + pf_reg(25), + pf_reg(26), + pf_reg(27), + pf_reg(28), + pf_reg(29), + pf_reg(30), + pf_reg(31), + ], + &[] + ], + scratch_by_class: [None, None, None], + fixed_stack_slots: &[], + } +}; From 9a1ef9ce7f6aa865bd8c853defec84696d24bb9b Mon Sep 17 00:00:00 2001 From: Jonas Kruckenberg Date: Mon, 13 Jan 2025 12:05:31 +0100 Subject: [PATCH 3/7] fixes --- cranelift/codegen/src/isa/pulley_shared/abi.rs | 3 +-- cranelift/codegen/src/isa/riscv64/abi.rs | 1 + cranelift/codegen/src/isa/x64/abi.rs | 2 +- cranelift/codegen/src/isa/x64/inst/regs.rs | 4 ++-- cranelift/codegen/src/machinst/reg.rs | 6 +++++- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cranelift/codegen/src/isa/pulley_shared/abi.rs b/cranelift/codegen/src/isa/pulley_shared/abi.rs index 02dcf836416e..4494cb617287 100644 --- a/cranelift/codegen/src/isa/pulley_shared/abi.rs +++ b/cranelift/codegen/src/isa/pulley_shared/abi.rs @@ -11,10 +11,9 @@ use crate::{ use alloc::{boxed::Box, vec::Vec}; use core::marker::PhantomData; use cranelift_bitset::ScalarBitSet; -use regalloc2::{MachineEnv, PReg, PRegSet}; +use regalloc2::{MachineEnv, PRegSet}; use smallvec::{smallvec, SmallVec}; use std::borrow::ToOwned; -use std::sync::OnceLock; /// Support for the Pulley ABI from the callee side (within a function body). pub(crate) type PulleyCallee

= Callee>; diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs index 91d9001cf9c0..58fe810f9803 100644 --- a/cranelift/codegen/src/isa/riscv64/abi.rs +++ b/cranelift/codegen/src/isa/riscv64/abi.rs @@ -19,6 +19,7 @@ use alloc::boxed::Box; use alloc::vec::Vec; use regalloc2::{MachineEnv, PRegSet}; use smallvec::{smallvec, SmallVec}; +use alloc::borrow::ToOwned; /// Support for the Riscv64 ABI from the callee side (within a function body). pub(crate) type Riscv64Callee = Callee; diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index d952e83ba79f..4d339d96faaa 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -1312,7 +1312,7 @@ const fn all_clobbers() -> PRegSet { .with(regs::fpr_preg(15)) } -fn preg(r: Reg) -> PReg { +const fn preg(r: Reg) -> PReg { r.to_physical_reg().unwrap() } diff --git a/cranelift/codegen/src/isa/x64/inst/regs.rs b/cranelift/codegen/src/isa/x64/inst/regs.rs index de7257d4d888..7df3b5e8d31b 100644 --- a/cranelift/codegen/src/isa/x64/inst/regs.rs +++ b/cranelift/codegen/src/isa/x64/inst/regs.rs @@ -33,7 +33,7 @@ pub const ENC_R15: u8 = 15; const fn gpr(enc: u8) -> Reg { let preg = gpr_preg(enc); - Reg::from(VReg::new(preg.index(), RegClass::Int)) + Reg::from_vreg(VReg::new(preg.index(), RegClass::Int)) } pub(crate) const fn gpr_preg(enc: u8) -> PReg { PReg::new(enc as usize, RegClass::Int) @@ -99,7 +99,7 @@ pub(crate) const fn pinned_reg() -> Reg { const fn fpr(enc: u8) -> Reg { let preg = fpr_preg(enc); - Reg::from(VReg::new(preg.index(), RegClass::Float)) + Reg::from_vreg(VReg::new(preg.index(), RegClass::Float)) } pub(crate) const fn fpr_preg(enc: u8) -> PReg { diff --git a/cranelift/codegen/src/machinst/reg.rs b/cranelift/codegen/src/machinst/reg.rs index ca11d82f9824..3a5c76600850 100644 --- a/cranelift/codegen/src/machinst/reg.rs +++ b/cranelift/codegen/src/machinst/reg.rs @@ -49,10 +49,14 @@ pub struct Reg(VReg); impl Reg { // FIXME(const-hack) remove in favor of `From` impl - pub(crate) const fn from_preg(preg: PReg) -> Reg { + pub const fn from_preg(preg: PReg) -> Reg { Reg(RealReg(preg).to_vreg()) } + pub const fn from_vreg(vreg: VReg) -> Reg { + Reg(vreg) + } + /// Get the physical register (`RealReg`), if this register is /// one. pub const fn to_real_reg(self) -> Option { From 846297f24f4c261bfba5041ce74fd7f492df21fb Mon Sep 17 00:00:00 2001 From: Jonas Kruckenberg Date: Mon, 13 Jan 2025 12:06:22 +0100 Subject: [PATCH 4/7] Update abi.rs --- cranelift/codegen/src/isa/x64/abi.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 4d339d96faaa..abe954d1e13d 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -1375,8 +1375,6 @@ static DEFAULT_MACHINE_ENV: MachineEnv = { }; static PINNED_MACHINE_ENV: MachineEnv = { - debug_assert!(regs::r15() == regs::pinned_reg()); - MachineEnv { preferred_regs_by_class: [ // Preferred GPRs: caller-saved in the SysV ABI. From beeb370e6ce066c8165816d4f94f788f90e115ba Mon Sep 17 00:00:00 2001 From: Jonas Kruckenberg Date: Mon, 13 Jan 2025 12:08:34 +0100 Subject: [PATCH 5/7] fixes --- cranelift/codegen/src/machinst/reg.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cranelift/codegen/src/machinst/reg.rs b/cranelift/codegen/src/machinst/reg.rs index 3a5c76600850..0df9e8ceea57 100644 --- a/cranelift/codegen/src/machinst/reg.rs +++ b/cranelift/codegen/src/machinst/reg.rs @@ -48,11 +48,13 @@ pub fn first_user_vreg_index() -> usize { pub struct Reg(VReg); impl Reg { + /// Create a register from a physical register // FIXME(const-hack) remove in favor of `From` impl pub const fn from_preg(preg: PReg) -> Reg { Reg(RealReg(preg).to_vreg()) } + /// Create a register from a virtual register pub const fn from_vreg(vreg: VReg) -> Reg { Reg(vreg) } From 080f09db9ef9cf4c3fa50f50b256b4df61f6e6f6 Mon Sep 17 00:00:00 2001 From: Jonas Kruckenberg Date: Mon, 13 Jan 2025 12:08:38 +0100 Subject: [PATCH 6/7] Update Cargo.toml --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 5da2cb346441..2ae461151b11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -564,3 +564,6 @@ opt-level = 's' inherits = "release" codegen-units = 1 lto = true + +[patch.crates-io] +regalloc2 = { git = "https://github.com/bytecodealliance/regalloc2", branch = "jonas/refactor/static-machine-env" } From e091faf42aec275ab9ce46b719bb79c8d2553a3c Mon Sep 17 00:00:00 2001 From: Jonas Kruckenberg Date: Mon, 13 Jan 2025 12:20:22 +0100 Subject: [PATCH 7/7] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2ae461151b11..060aae94dd34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -566,4 +566,4 @@ codegen-units = 1 lto = true [patch.crates-io] -regalloc2 = { git = "https://github.com/bytecodealliance/regalloc2", branch = "jonas/refactor/static-machine-env" } +regalloc2 = { git = "https://github.com/JonasKruckenberg/regalloc2", branch = "jonas/refactor/static-machine-env" }