diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 0262ea1085398..8a6d9b572dec0 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -82,6 +82,19 @@ impl WorkStorage { fence(Ordering::Release); Ok(()) } + /// Whether the firmware completed every kick counted so far on node `word`. + fn drained(&self, word: u32) -> Result { + let slot = (word & 63) as usize; + if self.words.get(slot) != Some(&Some(word)) { + return Err(EINVAL); + } + let completed = self + .object + .word(slot * NODE_STRIDE + core::mem::offset_of!(fw::WorkNode, completed))? + .load(Ordering::Relaxed); + fence(Ordering::Acquire); + Ok(completed >= self.counts[slot]) + } fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { let slot = (word & 63) as usize; if self.words.get(slot) != Some(&Some(word)) { @@ -476,6 +489,16 @@ impl WorkStateLease { } Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) } + /// Whether the firmware completed every kick counted so far on this ioctl's node. It stores + /// the completion stamps of the node's commands only then, so a command whose record is + /// present still waits for its stamp while other commands of its ioctl run. + pub(crate) fn drained(&self) -> Result { + let word = self.word(); + if word == FREE { + return Err(EINVAL); + } + self.context.work.lock().drained(word) + } pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { self.context .work diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index e1bda516ee102..bb2309641b88e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1025,6 +1025,7 @@ impl Queue { self.retire_pending = false; let mut oldest = true; while let Some(front) = self.active.front() { + let mut restart = None; let result = (|| { let ticket = front.ticket.ok_or(EIO)?; self.valid_ticket(front, &ticket)?; @@ -1044,7 +1045,18 @@ impl Queue { .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); let stamp = self.graph.stamp()?; - let record_aged = retirement::record_aged(front.observed_at, now_ns()); + let record_aged = !retirement::stamp_covers(stamp, ticket.kick) + && match retirement::stamp_wait(front.observed_at, now_ns(), || { + // A node that cannot be read keeps the bound from the record. + front.work.drained().unwrap_or(true) + }) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + restart = Some(now); + false + } + retirement::StampWait::Aged => true, + }; let timestamps = retirement::poll( &ticket, front.observed, @@ -1100,6 +1112,9 @@ impl Queue { } } Ok(None) | Err(EAGAIN) => { + if restart.is_some() { + self.active.iter_mut().next().ok_or(EIO)?.observed_at = restart; + } self.retire_pending = self.front_finished(); break; } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 6b942dcd86a3e..d7f821bbce68d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -72,15 +72,16 @@ pub(crate) enum Recordless { } /// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. +/// before it retires on the record alone, counted from when its ioctl's work-state node drained. /// /// The firmware stores the stamp after the cache maintenance that follows the record, so the /// stamp, not the record, proves that the command's writes are complete and that its mappings -/// may be released. While other commands of the same submit ioctl are still running, the stamp -/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms -/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as -/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail -/// its VM. +/// may be released. It stores the stamps of an ioctl's commands only once every kick counted on +/// the ioctl's work-state node has completed: while other commands of the same submit ioctl +/// still run, the stamp is not late, however long they run (a command buffer of a dozen render +/// passes, or a render waiting on another queue). [`stamp_wait`] restarts the bound until the +/// node drains. Only a stamp still missing well past that is treated as never stored. The bound +/// stays far below the scheduler timeout, so such a command cannot fail its VM. pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000; // A command that waits out the fallback must still retire well inside the scheduler timeout. @@ -92,6 +93,33 @@ pub(crate) fn record_aged(observed_at: Option, now: u64) -> bool { observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS) } +/// Where a command's wait for its completion stamp stands; see [`stamp_wait`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StampWait { + /// Within [`STAMP_FALLBACK_NS`] of the record, or no record yet. + Waiting, + /// The bound passed while the ioctl's work-state node still runs kicks: the stamp is not + /// late, and the wait counts again from the given host time. + Restart(u64), + /// The node drained and the stamp is still missing after the bound: retire on the record. + Aged, +} + +/// The stamp wait of a command whose exact record was latched at `observed_at` (host ns) and +/// whose stamp is still missing, at `now`. `drained` reports whether every kick counted on the +/// command's work-state node completed; it is asked only once the bound has passed. +pub(crate) fn stamp_wait(observed_at: Option, now: u64, drained: impl FnOnce() -> bool) + -> StampWait +{ + if !record_aged(observed_at, now) { + StampWait::Waiting + } else if drained() { + StampWait::Aged + } else { + StampWait::Restart(now) + } +} + static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); pub(crate) fn stampless_total() -> u64 { @@ -538,6 +566,22 @@ mod tests { assert_eq!(result, Ok(Some([0x1111, 0x2222]))); } + #[test] + fn a_stamp_waits_for_the_rest_of_its_ioctl() { + let at = 1_000_000; + let late = at + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(Some(at), late - 1, || panic!("node read early")), StampWait::Waiting); + assert_eq!(stamp_wait(None, u64::MAX, || panic!("node read without a record")), + StampWait::Waiting); + // Other kicks of the ioctl still run: the stamp is not late, the bound restarts. + assert_eq!(stamp_wait(Some(at), late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // The node drained and the stamp is still missing after the bound. + assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS, || true), StampWait::Aged); + assert_eq!(stamp_wait(Some(at), late, || true), StampWait::Aged); + } + #[test] fn an_aged_record_still_waits_for_consumed_items() { let record = Some(Observation { timestamps: [10, 20] }); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 2254118f18140..540f7cc76e8e7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1084,7 +1084,30 @@ impl Pair { return Ok(false); } let since = *front.stamp_wait_since.get_or_insert(now); - Ok(now.saturating_sub(since) >= retirement::STAMP_FALLBACK_NS) + Ok(Self::stamp_wait_expired(front, since, now)) + } + + /// Whether the front's stamp wait latched at `since` has run out by `now`. While other kicks + /// of the group's ioctl still run, its stamps are not late: the wait restarts and the pair + /// stays latched for the rescan. + fn stamp_wait_expired(front: &mut Active, since: u64, now: u64) -> bool { + let drained = || { + // A node that cannot be read keeps the bound from the latch. + front + .packet + .completion + .work_state() + .and_then(|work| work.drained()) + .unwrap_or(true) + }; + match retirement::stamp_wait(since, now, drained) { + retirement::StampWait::Waiting => false, + retirement::StampWait::Restart(now) => { + front.stamp_wait_since = Some(now); + false + } + retirement::StampWait::Expired => true, + } } /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair @@ -1322,9 +1345,9 @@ impl Pair { { return Err(EIO); } - let stampless = front.stamp_wait_since.is_some_and(|since| { - crate::g17::now_ns().saturating_sub(since) >= retirement::STAMP_FALLBACK_NS - }); + let stampless = front + .stamp_wait_since + .is_some_and(|since| Self::stamp_wait_expired(front, since, crate::g17::now_ns())); let mut observer = Observer { memory: &self.memory, qids, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index a62d291ea3d92..4377bbfe3e311 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -156,13 +156,17 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result Result bool) -> StampWait { + if now.saturating_sub(since) < STAMP_FALLBACK_NS { + StampWait::Waiting + } else if since == 0 || drained() { + StampWait::Expired + } else { + StampWait::Restart(now) + } +} + static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); /// Counts a render group that retired on its pass record because its completion stamps never @@ -484,4 +514,20 @@ mod executed_tests { let mut memory = stalled([10, 0]); assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); } + + #[test] + fn stamps_wait_for_the_rest_of_the_ioctl() { + let since = 1_000_000; + let late = since + STAMP_FALLBACK_NS; + // The node is asked only once the bound passed. + assert_eq!(stamp_wait(since, late - 1, || panic!("node read early")), StampWait::Waiting); + // Other kicks of the group's ioctl still run: the bound restarts. + assert_eq!(stamp_wait(since, late, || false), StampWait::Restart(late)); + assert_eq!(stamp_wait(late, late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting); + // A drained node with lagging stamps is the stall the bound exists for. + assert_eq!(stamp_wait(since, late, || true), StampWait::Expired); + // The watchdog's last resort does not wait for the node. + assert_eq!(stamp_wait(0, STAMP_FALLBACK_NS, || panic!("node read for an expired wait")), + StampWait::Expired); + } }