drm/asahi: retire a G17 render group on its pass record when its stamps stall (stacked on #244)

drivers/gpu/drm/asahi/g17/queue/render.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Physical paired tiling and fragment queues and their retained command storage.
@@ -142,6 +142,8 @@
     drained_result: Option<Result<[u64; 4]>>,
     /// First observation of this group retired and unlinked without execution.
     abandoned_since: Option<u64>,
+    /// First observation of this group executed and consumed while its stamps lagged.
+    stamp_wait_since: Option<u64>,
 }
 
 /// A firmware-retired, unexecuted group must hold on two observations this far
@@ -812,6 +814,7 @@
                 failed: false,
                 drained_result: None,
                 abandoned_since: None,
+                stamp_wait_since: None,
             })
             .map_err(|_| EBUSY)?;
         self.prepared = None;
@@ -1033,6 +1036,57 @@
             },
         ]
     }
+
+    /// Latches when the front group executed and the firmware consumed its ring entries while
+    /// `stamps` still lag it, and reports whether that has held for
+    /// [`retirement::STAMP_FALLBACK_NS`] (`now`, host ns). The observation scan then retires it
+    /// on its pass record instead of waiting for stamps no later event will move.
+    pub(crate) fn note_stamp_wait(&mut self, stamps: [u32; 2], state: u32, now: u64) -> Result<bool> {
+        if self.quarantined || self.prepared.is_some() || state != 0 {
+            return Ok(false);
+        }
+        let Some(front) = self.active.front_mut() else {
+            return Ok(false);
+        };
+        if !front.outer_published {
+            return Ok(false);
+        }
+        let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?;
+        if stamps.into_iter().all(|stamp| retirement::stamp_covers(stamp, published)) {
+            front.stamp_wait_since = None;
+            return Ok(false);
+        }
+        let snapshot = observation::snapshot(&self.memory, front.ticket.ordinal)?;
+        if !retirement::observe(&front.ticket, snapshot).is_ok_and(|seen| seen.complete) {
+            front.stamp_wait_since = None;
+            return Ok(false);
+        }
+        let since = *front.stamp_wait_since.get_or_insert(now);
+        Ok(now.saturating_sub(since) >= retirement::STAMP_FALLBACK_NS)
+    }
+
+    /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair
+    /// without a completion notification, which may never come for such a group.
+    pub(crate) fn stamp_waiting(&self) -> bool {
+        self.active.front().is_some_and(|front| front.stamp_wait_since.is_some())
+    }
+
+    /// The watchdog of `packet` expired while it is this pair's front: if the firmware executed
+    /// it and consumed its ring entries while the stamps lag, it retires on its pass record now
+    /// instead of failing. Failing would give up on the stamps as well.
+    pub(crate) fn expire_stamp_wait(&mut self, packet: &Arc<Packet>, stamps: [u32; 2], state: u32) -> Result<bool> {
+        if !self.active.front().is_some_and(|front| Arc::ptr_eq(&front.packet, packet)) {
+            return Ok(false);
+        }
+        if !self.note_stamp_wait(stamps, state, crate::g17::now_ns())? && !self.stamp_waiting() {
+            return Ok(false);
+        }
+        if let Some(front) = self.active.front_mut() {
+            front.stamp_wait_since = Some(0);
+        }
+        Ok(true)
+    }
+
     fn selected(&self, masks: Option<[u64; 2]>) -> bool {
         masks.is_none_or(|masks| completion_noted(self.qids(), masks))
     }
@@ -1246,6 +1300,9 @@
         {
             return Err(EIO);
         }
+        let stampless = front.stamp_wait_since.is_some_and(|since| {
+            crate::g17::now_ns().saturating_sub(since) >= retirement::STAMP_FALLBACK_NS
+        });
         let mut observer = Observer {
             memory: &self.memory,
             qids,
@@ -1255,7 +1312,13 @@
             if observed.ticket != front.ticket {
                 return Err(EIO);
             }
-            retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot)
+            if stampless {
+                retirement::poll_stampless(&mut front.ticket, &mut observer)
+            } else {
+                retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot)
+            }
+        } else if stampless {
+            retirement::poll_stampless(&mut front.ticket, &mut observer)
         } else {
             retirement::poll(&mut front.ticket, &mut observer)
         };
@@ -1270,6 +1333,9 @@
         };
         let ordinal = front.ticket.ordinal;
         let context_update = front.context_update;
+        if stampless {
+            retirement::note_stampless(u16::from(self.slot), ordinal);
+        }
         self.note_retirement(ordinal, context_update);
         let front = self.active.front_mut().ok_or(EIO)?;
         if !front.flist_completed {

drivers/gpu/drm/asahi/g17/queue/render/retirement.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Paired render retirement evidence and bounded ownership of in-flight work.
@@ -155,6 +155,55 @@
     })
 }
 
+/// How long a front group the firmware executed, and whose ring entries it consumed, waits for
+/// its completion stamp before it retires on its pass record.
+///
+/// The firmware stores both queues' stamps after the cache maintenance that follows a pass, so
+/// the stamps, not the pass record, normally prove completion. Occasionally the stamps of a pair
+/// stop one kick short while the pass record shows the group (and later groups) executed, and
+/// no later event moves them. Without a bound the client waits for the scheduler watchdog and
+/// its renewals, then loses its device. Compute applies the same bound to its records.
+///
+/// Nothing here proves that the maintenance finished: whether the firmware completes it before
+/// it advances the done cursors is not known. A group retired this way relies on this bound for
+/// its counts and stores to be visible, both for the fence and for the timestamps the host then
+/// writes (`completion::Output::publish()`).
+pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000;
+const _: () = assert!(STAMP_FALLBACK_NS * 4 <= crate::g17::job::TIMEOUT_MS as u64 * 1_000_000);
+
+static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0);
+
+/// Counts a render group that retired on its pass record because its completion stamps never
+/// covered it; logs the first and every 64th.
+pub(crate) fn note_stampless(slot: u16, ordinal: u64) {
+    let count = STAMPLESS.fetch_add(1, Ordering::Relaxed) + 1;
+    if count == 1 || count % 64 == 0 {
+        pr_info!(
+            "G17: lifetime total {} render groups retired on their pass record after {} ms without completion stamps (latest: slot {} ordinal {})\n",
+            count,
+            STAMP_FALLBACK_NS / 1_000_000,
+            slot,
+            ordinal
+        );
+    }
+}
+
+/// Retire an executed, consumed front group without its stamps, once [`STAMP_FALLBACK_NS`]
+/// has passed. The caller decides that bound; this re-checks execution and consumption. Unlike
+/// the stamped path, it has no post-maintenance evidence; see [`STAMP_FALLBACK_NS`].
+pub(crate) fn poll_stampless(ticket: &mut Ticket, memory: &mut impl Memory) -> Result<Poll> {
+    fence(Ordering::Acquire);
+    let snapshot = memory.snapshot(ticket.ordinal)?;
+    let Ok(observation) = observe(ticket, snapshot) else {
+        return Ok(Poll::Retry);
+    };
+    if !observation.complete {
+        return Ok(Poll::Pending);
+    }
+    let [start, end] = memory.pass(ticket.ordinal)?;
+    Ok(Poll::Complete([start, start, start, end]))
+}
+
 /// Outstanding publications span less than half the 24-bit stamp sequence.
 /// A zero stamp is valid when that sequence wraps.
 pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool {
@@ -382,4 +431,57 @@
         assert!(!snapshot(3, 3, [0, 2]).executed_and_consumed());
         assert!(!snapshot(2, 3, [1, 2]).executed_and_consumed());
     }
+
+    /// The firmware state of a stalled pair as read on hardware: the front group executed and
+    /// its two ring entries per queue were consumed, but both stamps hold the previous kick.
+    struct Stalled {
+        snapshot: Snapshot,
+        stamps: [u32; 2],
+    }
+    impl Memory for Stalled {
+        fn snapshot(&mut self, _ordinal: u64) -> Result<Snapshot> {
+            Ok(self.snapshot)
+        }
+        fn stamps(&mut self) -> Result<[u32; 2]> {
+            Ok(self.stamps)
+        }
+        fn pass(&mut self, _ordinal: u64) -> Result<[u64; 2]> {
+            Ok(self.snapshot.pass)
+        }
+    }
+
+    fn stalled_ticket() -> Ticket {
+        let window = Window::new(1097, 2, 1280).unwrap();
+        let previous = Cursors { done: 1097, read: 1097, write: 1097 };
+        Ticket { ordinal: 406, windows: [window; 2], previous: [previous; 2] }
+    }
+
+    fn stalled(pass: [u64; 2]) -> Stalled {
+        let cursors = Cursors { done: 1099, read: 1099, write: 1099 };
+        Stalled {
+            snapshot: Snapshot { cursors: [cursors; 2], job_list_empty: false, pass },
+            stamps: [406 << 8; 2],
+        }
+    }
+
+    #[test]
+    fn lagging_stamps_keep_an_executed_group_pending() {
+        let mut ticket = stalled_ticket();
+        let mut memory = stalled([10, 20]);
+        assert!(matches!(poll(&mut ticket, &mut memory), Ok(Poll::Pending)));
+    }
+
+    #[test]
+    fn stampless_poll_retires_an_executed_consumed_group() {
+        let mut ticket = stalled_ticket();
+        let mut memory = stalled([10, 20]);
+        assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Complete([10, 10, 10, 20]))));
+    }
+
+    #[test]
+    fn stampless_poll_waits_for_the_fragment_end() {
+        let mut ticket = stalled_ticket();
+        let mut memory = stalled([10, 0]);
+        assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending)));
+    }
 }

drivers/gpu/drm/asahi/g17/runtime/render.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Device ownership of retained render pairs and their shared free-list leases.
@@ -798,6 +798,19 @@
             {
                 continue;
             }
+            let state = crate::g17::recovery::Memory::recovery_state(&self.init)?;
+            if let Some(pair) = self.queues.render.entries[slot]
+                .as_mut()
+                .and_then(|entry| entry.pair.as_mut())
+            {
+                let qids = pair.qids();
+                let stamps = [
+                    self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed),
+                    self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed),
+                ];
+                core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire);
+                pair.note_stamp_wait(stamps, state, crate::g17::now_ns())?;
+            }
             self.retire_render_slot(slot, None, deferred)?;
             self.queues.render.collect_retired(slot)?;
             self.settle_render_terminal(slot, deferred)?;
@@ -1057,10 +1070,10 @@
                     |pair| pair.published_in_flight(),
                 );
             active
-                && crate::g17::queue::render::completion_noted(
+                && (crate::g17::queue::render::completion_noted(
                     entry.reservation.ids.map(kick::Id::qid),
                     entry.event_masks,
-                )
+                ) || entry.pair.as_ref().is_some_and(|pair| pair.stamp_waiting()))
         })
     }
     pub(in crate::g17) fn defer_render_timeout(

drivers/gpu/drm/asahi/g17/runtime/render/observe.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Event-driven render snapshots. A pair's completion memory is read and the
@@ -10,7 +10,7 @@
 impl crate::g17::Firmware {
     /// The first slot from `start` whose pair has news to observe: published work in flight, the
     /// fragment queue's notification seen (`completion_noted`), and an observation the pair accepts.
-    fn next_render_observation(&self, start: usize) -> Result<Option<usize>> {
+    fn next_render_observation(&mut self, start: usize) -> Result<Option<usize>> {
         if self.recovery.pending() {
             return Ok(None);
         }
@@ -27,7 +27,8 @@
             };
             let qids = pair.qids();
             if !pair.published_in_flight()
-                || !crate::g17::queue::render::completion_noted(qids, entry.event_masks)
+                || (!crate::g17::queue::render::completion_noted(qids, entry.event_masks)
+                    && !pair.stamp_waiting())
             {
                 continue;
             }
@@ -39,6 +40,15 @@
             if pair.observation_packet(entry.event_masks, stamps, state)?.is_some() {
                 return Ok(Some(slot));
             }
+            // The stamps gate this pair out. If its front executed and was consumed anyway, the
+            // stamps may never move: after the bound, observe it and retire it on its record.
+            let pair = self.queues.render.entries[slot]
+                .as_mut()
+                .and_then(|entry| entry.pair.as_mut())
+                .ok_or(EIO)?;
+            if pair.note_stamp_wait(stamps, state, crate::g17::now_ns())? {
+                return Ok(Some(slot));
+            }
         }
         Ok(None)
     }

drivers/gpu/drm/asahi/g17/runtime/render/settle.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Exact packet timeout and paired firmware-drain settlement. Every fence and
@@ -79,6 +79,20 @@
             }
             let generation = self.recovery.generation();
             let state = crate::g17::recovery::Memory::recovery_state(&self.init)?;
+            if error == ETIMEDOUT {
+                let qids = pair.qids();
+                let stamps = [
+                    self.init.stamp(qids[0])?.load(core::sync::atomic::Ordering::Relaxed),
+                    self.init.stamp(qids[1])?.load(core::sync::atomic::Ordering::Relaxed),
+                ];
+                core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire);
+                if let Some(pair) = self.queues.render.entries[slot]
+                    .as_mut()
+                    .and_then(|entry| entry.pair.as_mut())
+                {
+                    pair.expire_stamp_wait(packet, stamps, state)?;
+                }
+            }
             let progressed = self.with_render(slot, |pair, host| {
                 pair.retire(None, generation, state, host, &mut |event| {
                     deferred.push(event)