diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 3261f2f0640b32..6fd769564847a1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -142,6 +142,8 @@ struct Active { drained_result: Option>, /// First observation of this group retired and unlinked without execution. abandoned_since: Option, + /// First observation of this group executed and consumed while its stamps lagged. + stamp_wait_since: Option, } /// A firmware-retired, unexecuted group must hold on two observations this far @@ -812,6 +814,7 @@ impl Pair { failed: false, drained_result: None, abandoned_since: None, + stamp_wait_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -1033,6 +1036,57 @@ impl Pair { }, ] } + + /// Latches when the front group executed and the firmware consumed its ring entries while + /// `stamps` still lag it, and reports whether that has held for + /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it + /// on its pass record instead of waiting for stamps no later event will move. + pub(crate) fn note_stamp_wait(&mut self, stamps: [u32; 2], state: u32, now: u64) -> Result { + if self.quarantined || self.prepared.is_some() || state != 0 { + return Ok(false); + } + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + if stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) { + front.stamp_wait_since = None; + return Ok(false); + } + let snapshot = observation::snapshot(&self.memory, front.ticket.ordinal)?; + if !retirement::observe(&front.ticket, snapshot).is_ok_and(|seen| seen.complete) { + front.stamp_wait_since = None; + return Ok(false); + } + let since = *front.stamp_wait_since.get_or_insert(now); + Ok(now.saturating_sub(since) >= retirement::STAMP_FALLBACK_NS) + } + + /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair + /// without a completion notification, which may never come for such a group. + pub(crate) fn stamp_waiting(&self) -> bool { + self.active.front().is_some_and(|front| front.stamp_wait_since.is_some()) + } + + /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed + /// it and consumed its ring entries while the stamps lag, it retires on its pass record now + /// instead of failing. Failing would give up on the stamps as well. + pub(crate) fn expire_stamp_wait(&mut self, packet: &Arc, stamps: [u32; 2], state: u32) -> Result { + if !self.active.front().is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) { + return Ok(false); + } + if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() { + return Ok(false); + } + if let Some(front) = self.active.front_mut() { + front.stamp_wait_since = Some(0); + } + Ok(true) + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { masks.is_none_or(|masks| completion_noted(self.qids(), masks)) } @@ -1246,6 +1300,9 @@ impl Pair { { return Err(EIO); } + let stampless = front.stamp_wait_since.is_some_and(|since| { + crate::g17::now_ns().saturating_sub(since) >= retirement::STAMP_FALLBACK_NS + }); let mut observer = Observer { memory: &self.memory, qids, @@ -1255,7 +1312,13 @@ impl Pair { if observed.ticket != front.ticket { return Err(EIO); } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + if stampless { + retirement::poll_stampless(&mut front.ticket, &mut observer) + } else { + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } + } else if stampless { + retirement::poll_stampless(&mut front.ticket, &mut observer) } else { retirement::poll(&mut front.ticket, &mut observer) }; @@ -1270,6 +1333,9 @@ impl Pair { }; let ordinal = front.ticket.ordinal; let context_update = front.context_update; + if stampless { + retirement::note_stampless(u16::from(self.slot), ordinal); + } self.note_retirement(ordinal, context_update); let front = self.active.front_mut().ok_or(EIO)?; if !front.flist_completed { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 7996970ee369ff..a62d291ea3d921 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -155,6 +155,55 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result Result { + fence(Ordering::Acquire); + let snapshot = memory.snapshot(ticket.ordinal)?; + let Ok(observation) = observe(ticket, snapshot) else { + return Ok(Poll::Retry); + }; + if !observation.complete { + return Ok(Poll::Pending); + } + let [start, end] = memory.pass(ticket.ordinal)?; + Ok(Poll::Complete([start, start, start, end])) +} + /// Outstanding publications span less than half the 24-bit stamp sequence. /// A zero stamp is valid when that sequence wraps. pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { @@ -382,4 +431,57 @@ mod executed_tests { assert!(!snapshot(3, 3, [0, 2]).executed_and_consumed()); assert!(!snapshot(2, 3, [1, 2]).executed_and_consumed()); } + + /// The firmware state of a stalled pair as read on hardware: the front group executed and + /// its two ring entries per queue were consumed, but both stamps hold the previous kick. + struct Stalled { + snapshot: Snapshot, + stamps: [u32; 2], + } + impl Memory for Stalled { + fn snapshot(&mut self, _ordinal: u64) -> Result { + Ok(self.snapshot) + } + fn stamps(&mut self) -> Result<[u32; 2]> { + Ok(self.stamps) + } + fn pass(&mut self, _ordinal: u64) -> Result<[u64; 2]> { + Ok(self.snapshot.pass) + } + } + + fn stalled_ticket() -> Ticket { + let window = Window::new(1097, 2, 1280).unwrap(); + let previous = Cursors { done: 1097, read: 1097, write: 1097 }; + Ticket { ordinal: 406, windows: [window; 2], previous: [previous; 2] } + } + + fn stalled(pass: [u64; 2]) -> Stalled { + let cursors = Cursors { done: 1099, read: 1099, write: 1099 }; + Stalled { + snapshot: Snapshot { cursors: [cursors; 2], job_list_empty: false, pass }, + stamps: [406 << 8; 2], + } + } + + #[test] + fn lagging_stamps_keep_an_executed_group_pending() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 20]); + assert!(matches!(poll(&mut ticket, &mut memory), Ok(Poll::Pending))); + } + + #[test] + fn stampless_poll_retires_an_executed_consumed_group() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 20]); + assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Complete([10, 10, 10, 20])))); + } + + #[test] + fn stampless_poll_waits_for_the_fragment_end() { + let mut ticket = stalled_ticket(); + let mut memory = stalled([10, 0]); + assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending))); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 4e7eaa2d8d2468..6657f83e9e738a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -798,6 +798,19 @@ impl crate::g17::Firmware { { continue; } + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if let Some(pair) = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + { + let qids = pair.qids(); + let stamps = [ + self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), + self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), + ]; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + pair.note_stamp_wait(stamps, state, crate::g17::now_ns())?; + } self.retire_render_slot(slot, None, deferred)?; self.queues.render.collect_retired(slot)?; self.settle_render_terminal(slot, deferred)?; @@ -1057,10 +1070,10 @@ impl Registry { |pair| pair.published_in_flight(), ); active - && crate::g17::queue::render::completion_noted( + && (crate::g17::queue::render::completion_noted( entry.reservation.ids.map(kick::Id::qid), entry.event_masks, - ) + ) || entry.pair.as_ref().is_some_and(|pair| pair.stamp_waiting())) }) } pub(in crate::g17) fn defer_render_timeout( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 51682b43728c05..57543be54e8356 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -10,7 +10,7 @@ use kernel::{prelude::*, sync::Arc}; impl crate::g17::Firmware { /// The first slot from `start` whose pair has news to observe: published work in flight, the /// fragment queue's notification seen (`completion_noted`), and an observation the pair accepts. - fn next_render_observation(&self, start: usize) -> Result> { + fn next_render_observation(&mut self, start: usize) -> Result> { if self.recovery.pending() { return Ok(None); } @@ -27,7 +27,8 @@ impl crate::g17::Firmware { }; let qids = pair.qids(); if !pair.published_in_flight() - || !crate::g17::queue::render::completion_noted(qids, entry.event_masks) + || (!crate::g17::queue::render::completion_noted(qids, entry.event_masks) + && !pair.stamp_waiting()) { continue; } @@ -39,6 +40,15 @@ impl crate::g17::Firmware { if pair.observation_packet(entry.event_masks, stamps, state)?.is_some() { return Ok(Some(slot)); } + // The stamps gate this pair out. If its front executed and was consumed anyway, the + // stamps may never move: after the bound, observe it and retire it on its record. + let pair = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .ok_or(EIO)?; + if pair.note_stamp_wait(stamps, state, crate::g17::now_ns())? { + return Ok(Some(slot)); + } } Ok(None) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 24c724cb32470b..71f0bfc0526968 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -79,6 +79,20 @@ impl crate::g17::Firmware { } let generation = self.recovery.generation(); let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if error == ETIMEDOUT { + let qids = pair.qids(); + let stamps = [ + self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed), + self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed), + ]; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + if let Some(pair) = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + { + pair.expire_stamp_wait(packet, stamps, state)?; + } + } let progressed = self.with_render(slot, |pair, host| { pair.retire(None, generation, state, host, &mut |event| { deferred.push(event)