From 7eaf9b669b9ba568195d85c0c38cd0bd3886ca1f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 29 Oct 2026 07:39:19 -0400 Subject: [PATCH] drm/asahi: g17: count the stamp fallback from the drain of the command's node The firmware stores the completion stamps of an ioctl's commands only once every kick counted on the ioctl's work-state node has completed: a node's completion word reaches the host's submitted count, and only then do the compute and render stamps move. A command whose record or pass record is present therefore waits for its stamp for as long as the rest of its ioctl runs. A command buffer of a dozen render passes and their copies, or a compute command beside a render that waits on another queue, keeps the stamps of its early members back for tens to hundreds of milliseconds. The 50 ms stampless fallback measured from the record, so it retired those members on their records while the firmware was still running their node: their fences, their VM pins and their render leases were released before the post-maintenance stamp the retirement path otherwise requires. On the Neo, one CTS synchronization2 pass retired about 9.8k compute commands this way. Each of them had kicks of its own node still running (the node's completion word below the host count), and the stamp covered it 4 to 300 ms later (median 28 ms). Count the bound from the drain instead. Once it has passed, a compute command or a render group whose stamps lag asks its node whether every counted kick completed; if not, the stamps are not late and the wait starts again. A drained node whose stamps still lag keeps the existing fallback, so a group whose stamps stop one kick short still retires on its pass record, and the watchdog's expiry still retires an executed group without waiting for the node. A node that cannot be read keeps the old bound. Signed-off-by: DjDeveloperr (cherry picked from commit effb9e00034d83ceb6ac483ee448826d87bc4526) --- drivers/gpu/drm/asahi/g17/context.rs | 23 ++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 17 +++++- .../drm/asahi/g17/queue/compute/retirement.rs | 56 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/queue/render.rs | 31 ++++++++-- .../drm/asahi/g17/queue/render/retirement.rs | 56 +++++++++++++++++-- 5 files changed, 167 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 0b3bd97fb42fe4..409e5a3dcad9fc 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -81,6 +81,19 @@ impl WorkStorage { fence(Ordering::Release); Ok(()) } + /// Whether the firmware completed every kick counted so far on node `word`. + fn drained(&self, word: u32) -> Result { + let slot = (word & 63) as usize; + if self.words.get(slot) != Some(&Some(word)) { + return Err(EINVAL); + } + let completed = self + .object + .word(slot * NODE_STRIDE + core::mem::offset_of!(fw::WorkNode, completed))? + .load(Ordering::Relaxed); + fence(Ordering::Acquire); + Ok(completed >= self.counts[slot]) + } fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { let slot = (word & 63) as usize; if self.words.get(slot) != Some(&Some(word)) { @@ -448,6 +461,16 @@ impl WorkStateLease { } Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) } + /// Whether the firmware completed every kick counted so far on this ioctl's node. It stores + /// the completion stamps of the node's commands only then, so a command whose record is + /// present still waits for its stamp while other commands of its ioctl run. + pub(crate) fn drained(&self) -> Result { + let word = self.word(); + if word == FREE { + return Err(EINVAL); + } + self.context.work.lock().drained(word) + } pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { self.context .work diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index e1bda516ee1022..bb2309641b88ef 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1025,6 +1025,7 @@ impl Queue { self.retire_pending = false; let mut oldest = true; while let Some(front) = self.active.front() { + let mut restart = None; let result = (|| { let ticket = front.ticket.ok_or(EIO)?; self.valid_ticket(front, &ticket)?; @@ -1044,7 +1045,18 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = retirement::record_aged(front.observed_at, now_ns()); + let record_aged = !retirement::stamp_covers(stamp, ticket.kick) + && match retirement::stamp_wait(front.observed_at, now_ns(), || { + // A node that cannot be read keeps the bound from the record. + front.work.drained().unwrap_or(true) + }) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + restart = Some(now); + false + } + retirement::StampWait::Aged => true, + }; let timestamps = retirement::poll( &ticket, front.observed, @@ -1100,6 +1112,9 @@ impl Queue { } } Ok(None) | Err(EAGAIN) => { + if restart.is_some() { + self.active.iter_mut().next().ok_or(EIO)?.observed_at = restart; + } self.retire_pending = self.front_finished(); break; } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 8a737546cb7cf9..34c517aaa58457 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -72,15 +72,16 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. +/// before it retires on the record alone, counted from when its ioctl's work-state node drained. /// /// The firmware stores the stamp after the cache maintenance that follows the record, so the /// stamp, not the record, proves that the command's writes are complete and that its mappings -/// may be released. While other commands of the same submit ioctl are still running, the stamp -/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms -/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as -/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail -/// its VM. +/// may be released. It stores the stamps of an ioctl's commands only once every kick counted on +/// the ioctl's work-state node has completed: while other commands of the same submit ioctl +/// still run, the stamp is not late, however long they run (a command buffer of a dozen render +/// passes, or a render waiting on another queue). [`stamp_wait`] restarts the bound until the +/// node drains. Only a stamp still missing well past that is treated as never stored. The bound +/// stays far below the scheduler timeout, so such a command cannot fail its VM. pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; // A command that waits out the fallback must still retire well inside the scheduler timeout. @@ -92,6 +93,33 @@ pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) } +/// Where a command's wait for its completion stamp stands; see [`stamp_wait`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StampWait { + /// Within [`STAMP_FALLBACK_NS`] of the record, or no record yet. + Waiting, + /// The bound passed while the ioctl's work-state node still runs kicks: the stamp is not + /// late, and the wait counts again from the given host time. + Restart(u64), + /// The node drained and the stamp is still missing after the bound: retire on the record. + Aged, +} + +/// The stamp wait of a command whose exact record was latched at `observed_at` (host ns) and +/// whose stamp is still missing, at `now`. `drained` reports whether every kick counted on the +/// command's work-state node completed; it is asked only once the bound has passed. +pub(crate) fn stamp_wait(observed_at: Option, now: u64, drained: impl FnOnce() -> bool) + -> StampWait +{ + if !record_aged(observed_at, now) { + StampWait::Waiting + } else if drained() { + StampWait::Aged + } else { + StampWait::Restart(now) + } +} + static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); /// Counts a command that retired on its record because its completion stamp never arrived; @@ -530,6 +558,22 @@ mod tests { assert_eq!(result, Ok(Some([0x1111, 0x2222]))); } + #[test] + fn a_stamp_waits_for_the_rest_of_its_ioctl() { + let at = 1_000_000; + let late = at + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(Some(at), late - 1, || panic!("node read early")), StampWait::Waiting); + assert_eq!(stamp_wait(None, u64::MAX, || panic!("node read without a record")), + StampWait::Waiting); + // Other kicks of the ioctl still run: the stamp is not late, the bound restarts. + assert_eq!(stamp_wait(Some(at), late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // The node drained and the stamp is still missing after the bound. + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS, || true), StampWait::Aged); + assert_eq!(stamp_wait(Some(at), late, || true), StampWait::Aged); + } + #[test] fn an_aged_record_still_waits_for_consumed_items() { let record = Some(Observation { timestamps: [10, 20] }); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 6fd769564847a1..475394d1c3e1ab 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1062,7 +1062,30 @@ impl Pair { return Ok(false); } let since = *front.stamp_wait_since.get_or_insert(now); - Ok(now.saturating_sub(since) >= retirement::STAMP_FALLBACK_NS) + Ok(Self::stamp_wait_expired(front, since, now)) + } + + /// Whether the front's stamp wait latched at `since` has run out by `now`. While other kicks + /// of the group's ioctl still run, its stamps are not late: the wait restarts and the pair + /// stays latched for the rescan. + fn stamp_wait_expired(front: &mut Active, since: u64, now: u64) -> bool { + let drained = || { + // A node that cannot be read keeps the bound from the latch. + front + .packet + .completion + .work_state() + .and_then(|work| work.drained()) + .unwrap_or(true) + }; + match retirement::stamp_wait(since, now, drained) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + front.stamp_wait_since = Some(now); + false + } + retirement::StampWait::Expired => true, + } } /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair @@ -1300,9 +1323,9 @@ impl Pair { { return Err(EIO); } - let stampless = front.stamp_wait_since.is_some_and(|since| { - crate::g17::now_ns().saturating_sub(since) >= retirement::STAMP_FALLBACK_NS - }); + let stampless = front + .stamp_wait_since + .is_some_and(|since| Self::stamp_wait_expired(front, since, crate::g17::now_ns())); let mut observer = Observer { memory: &self.memory, qids, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index a62d291ea3d921..4377bbfe3e311d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -156,13 +156,17 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result Result bool) -> StampWait { + if now.saturating_sub(since) < STAMP_FALLBACK_NS { + StampWait::Waiting + } else if since == 0 || drained() { + StampWait::Expired + } else { + StampWait::Restart(now) + } +} + static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); /// Counts a render group that retired on its pass record because its completion stamps never @@ -484,4 +514,20 @@ mod executed_tests { let mut memory = stalled([10, 0]); assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); } + + #[test] + fn stamps_wait_for_the_rest_of_the_ioctl() { + let since = 1_000_000; + let late = since + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(since, late - 1, || panic!("node read early")), StampWait::Waiting); + // Other kicks of the group's ioctl still run: the bound restarts. + assert_eq!(stamp_wait(since, late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(late, late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // A drained node with lagging stamps is the stall the bound exists for. + assert_eq!(stamp_wait(since, late, || true), StampWait::Expired); + // The watchdog's last resort does not wait for the node. + assert_eq!(stamp_wait(0, STAMP_FALLBACK_NS, || panic!("node read for an expired wait")), + StampWait::Expired); + } }