diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index a40358d1be3c6f..0262ea10853983 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -15,7 +15,7 @@ use super::{ }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering}; -use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; use super::job::Engine; use kernel::{ new_mutex, @@ -128,6 +128,11 @@ pub(crate) struct Context { states: WorkStates, /// Signalled when a work state returns; publishers that found none wait on it. state_returns: Arc, + /// The domain refused compute publishers wait on; signalled with `state_returns` when the + /// admission turn moves on. + compute_returns: Arc, + /// The order in which this context's ioctls take work-state nodes. + order: Arc>, work: Arc>, work_base: u64, render: Arc>>>, @@ -139,6 +144,10 @@ pub(crate) struct Context { policy: Policy, } impl Context { + #[expect( + clippy::too_many_arguments, + reason = "A context takes its allocators, identities and both admission domains." + )] pub(crate) fn new( alloc: &Allocator<'_>, pool: &Arc, @@ -147,6 +156,7 @@ impl Context { owner_pid: u32, policy: Policy, state_returns: Arc, + compute_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -195,6 +205,8 @@ impl Context { GFP_KERNEL, )?; let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; + let order = + Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?; Ok(Self { execution, status, @@ -208,6 +220,8 @@ impl Context { owner_pid, states: WorkStates::new(), state_returns, + compute_returns, + order, work, work_base, render, @@ -399,18 +413,23 @@ pub(crate) struct WorkStateLease { word: AtomicU32, /// Index of the engine whose admission took the node, valid once `word` is set. engine: AtomicU8, + /// The ioctl's place in its context's submission order. + ordinal: u64, } impl WorkStateLease { pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { + let ordinal = context.order.lock().assign(); Self { context, word: AtomicU32::new(FREE), engine: AtomicU8::new(0), + ordinal, } } - /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: none - /// may be taken now (see `identity::may_take`); a later return makes one available. + /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many + /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now + /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again. pub(crate) fn acquire(&self, engine: Engine) -> Result { let current = self.word(); if current != FREE { @@ -423,6 +442,10 @@ impl WorkStateLease { if current != FREE { return Ok(current); } + let mut order = self.context.order.lock(); + if !order.admits(self.ordinal) { + return Err(EBUSY); + } let engine = engine.index(); let candidate = self.context.states.allocate(engine).ok_or(EBUSY)?; if let Err(error) = work.initialize(candidate) { @@ -431,6 +454,12 @@ impl WorkStateLease { } self.engine.store(engine as u8, Ordering::Relaxed); self.word.store(candidate, Ordering::Release); + let wake = order.admit(self.ordinal); + drop(order); + drop(work); + if wake { + self.context.pass_turn(); + } Ok(candidate) } /// The domain a publisher samples before acquire() and waits on after EBUSY. @@ -478,6 +507,8 @@ impl Drop for WorkStateLease { let word = self.word(); if word != FREE { self.context.release_state(word, usize::from(self.engine.load(Ordering::Relaxed))); + } else if self.context.order.lock().leave(self.ordinal) { + self.context.pass_turn(); } } } @@ -490,6 +521,12 @@ impl Context { self.state_returns.notify(); } } + /// A refused publisher waits on its engine's domain. Bumping both once the turn moved past a + /// refusal means one that sampled before its refused acquire() never sleeps through it. + fn pass_turn(&self) { + self.state_returns.notify(); + self.compute_returns.notify(); + } } impl QosIds { diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs index 9162e3e6bc4a4c..7586f8474e602f 100644 --- a/drivers/gpu/drm/asahi/g17/context/identity.rs +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -77,6 +77,101 @@ impl WorkStates { } } +/// Ordinals past the turn remembered for ioctls that took a node or left early. +pub(super) const ORDER_WINDOW: u64 = 1024; + +/// How far past the oldest waiting ioctl an ioctl may take its node. +pub(super) const RUN_AHEAD: u64 = 8; + +/// The order in which a context's ioctls take work-state nodes. +/// +/// Each engine admits a context's ioctls in submission order, but the engines advance +/// independently: render admission can take nodes for ioctls far ahead of an older ioctl still +/// waiting for its turn on the compute queue. Under deep mixed load (128 compute and 64 render +/// ioctls in flight on one queue) the firmware then stops completing all of the context's work +/// within seconds. Native macOS schedules a queue's command buffers in commit order across +/// engines, at most 36 between scheduling and completion, and the same load runs there without +/// failure. +/// +/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have +/// neither taken one nor left without one. Native scheduling happens at commit, on one thread; +/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler +/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that +/// without letting one engine get far ahead of the other. +pub(super) struct Order { + /// The next ordinal to hand out. + next: u64, + /// The oldest ordinal that has neither taken a node nor left. + turn: u64, + /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW. + done: [u64; (ORDER_WINDOW / 64) as usize], + /// Set once an ioctl left too far past the turn to remember; admission is unordered after. + unordered: bool, + /// An ioctl was refused since the turn last moved: its publisher waits for the next move. + refused: bool, +} +impl Order { + pub(super) const fn new() -> Self { + Self { + next: 0, + turn: 0, + done: [0; (ORDER_WINDOW / 64) as usize], + unordered: false, + refused: false, + } + } + /// The ordinal of a new ioctl, in submission order. + pub(super) fn assign(&mut self) -> u64 { + let ordinal = self.next; + self.next += 1; + ordinal + } + /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the + /// turn next moves, so that move wakes the refused publisher. + pub(super) fn admits(&mut self, ordinal: u64) -> bool { + let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD; + self.refused |= !admits; + admits + } + /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted. + pub(super) fn admit(&mut self, ordinal: u64) -> bool { + self.finish(ordinal) + } + /// The ioctl `ordinal` left without taking a node (it failed or was cancelled first). + /// Returns whether a refused ioctl may now be admitted. + pub(super) fn leave(&mut self, ordinal: u64) -> bool { + self.finish(ordinal) + } + fn finish(&mut self, ordinal: u64) -> bool { + if self.unordered || ordinal < self.turn { + return false; + } + if ordinal == self.turn { + self.pass(); + return core::mem::take(&mut self.refused); + } + if ordinal - self.turn >= ORDER_WINDOW { + self.unordered = true; + return core::mem::take(&mut self.refused); + } + let bit = ordinal % ORDER_WINDOW; + self.done[(bit / 64) as usize] |= 1 << (bit % 64); + false + } + /// Moves the turn past the current ordinal and every following one already done. + fn pass(&mut self) { + loop { + self.turn += 1; + let bit = self.turn % ORDER_WINDOW; + let (word, mask) = ((bit / 64) as usize, 1u64 << (bit % 64)); + if self.done[word] & mask == 0 { + break; + } + self.done[word] &= !mask; + } + } +} + /// Two stable logical ordinals, independent of physical pair numbering. pub(super) struct RenderSlots { pairs: [Option; 2], @@ -154,6 +249,71 @@ mod tests { assert!(states.allocate(1).is_some()); } + #[test] + fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() { + let mut order = Order::new(); + let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign()); + let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]); + // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node. + for &ordinal in &ordinals[..=RUN_AHEAD as usize] { + assert!(order.admits(ordinal)); + } + assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); + // Ioctls ahead of the turn taking their nodes do not move it. + for &ordinal in &ordinals[1..=RUN_AHEAD as usize] { + assert!(!order.admit(ordinal)); + } + assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1])); + // The oldest one moves the turn past everything already admitted, and wakes the refused. + assert!(order.admit(oldest)); + assert!(order.admits(last)); + let next = order.assign(); + assert!(order.admits(next)); + assert!(!order.admits(next + RUN_AHEAD + 1)); + } + + #[test] + fn ioctls_leaving_without_a_node_pass_the_turn() { + let mut order = Order::new(); + let (a, b, c) = (order.assign(), order.assign(), order.assign()); + let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign()); + let beyond = later[RUN_AHEAD as usize - 1]; + // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too. + assert!(!order.leave(b)); + assert!(!order.leave(c)); + assert!(!order.admits(beyond)); + // Once a takes its node, the turn passes b and c as well, and the refused one may retry. + assert!(order.admit(a)); + assert!(order.admits(beyond)); + // With nobody refused since, moving the turn wakes nobody; an ordinal already passed + // leaves without effect. + assert!(!order.leave(later[0])); + assert!(!order.leave(a)); + } + + #[test] + fn an_ioctl_leaving_too_far_ahead_unorders_admission() { + let mut order = Order::new(); + let first = order.assign(); + let mut last = first; + for _ in 0..ORDER_WINDOW { + last = order.assign(); + } + assert!(!order.admits(last)); + assert!(order.leave(last)); + assert!(order.admits(last - 1)); + } + + #[test] + fn single_engine_admission_wakes_nobody() { + let mut order = Order::new(); + for _ in 0..3 * RUN_AHEAD { + let ordinal = order.assign(); + assert!(order.admits(ordinal)); + assert!(!order.admit(ordinal)); + } + } + #[test] fn releases_return_nodes_to_their_engine() { let states = WorkStates::new(); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index eb0396745f3da6..0d1e390f19bb6a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -754,7 +754,16 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy, shared.renders.clone())?, + Context::new( + &alloc, + &pool, + &qos, + execution, + owner_pid, + policy, + shared.renders.clone(), + shared.progress.clone(), + )?, GFP_KERNEL, )?; let state = shared.state.lock();