drm/asahi: g17: count the stamp fallback from the drain of the command's node

drivers/gpu/drm/asahi/g17/context.rs

@@ -82,6 +82,19 @@ impl WorkStorage {
         fence(Ordering::Release);
         Ok(())
     }
+    /// Whether the firmware completed every kick counted so far on node `word`.
+    fn drained(&self, word: u32) -> Result<bool> {
+        let slot = (word & 63) as usize;
+        if self.words.get(slot) != Some(&Some(word)) {
+            return Err(EINVAL);
+        }
+        let completed = self
+            .object
+            .word(slot * NODE_STRIDE + core::mem::offset_of!(fw::WorkNode, completed))?
+            .load(Ordering::Relaxed);
+        fence(Ordering::Acquire);
+        Ok(completed >= self.counts[slot])
+    }
     fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result<u32> {
         let slot = (word & 63) as usize;
         if self.words.get(slot) != Some(&Some(word)) {
@@ -476,6 +489,16 @@ impl WorkStateLease {
         }
         Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64)
     }
+    /// Whether the firmware completed every kick counted so far on this ioctl's node. It stores
+    /// the completion stamps of the node's commands only then, so a command whose record is
+    /// present still waits for its stamp while other commands of its ioctl run.
+    pub(crate) fn drained(&self) -> Result<bool> {
+        let word = self.word();
+        if word == FREE {
+            return Err(EINVAL);
+        }
+        self.context.work.lock().drained(word)
+    }
     pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result<u32> {
         self.context
             .work

drivers/gpu/drm/asahi/g17/queue/compute.rs

@@ -1025,6 +1025,7 @@ impl Queue {
         self.retire_pending = false;
         let mut oldest = true;
         while let Some(front) = self.active.front() {
+            let mut restart = None;
             let result = (|| {
                 let ticket = front.ticket.ok_or(EIO)?;
                 self.valid_ticket(front, &ticket)?;
@@ -1044,7 +1045,18 @@ impl Queue {
                     .skip(1)
                     .find_map(|a| a.observed.map(|o| o.timestamps[1]));
                 let stamp = self.graph.stamp()?;
-                let record_aged = retirement::record_aged(front.observed_at, now_ns());
+                let record_aged = !retirement::stamp_covers(stamp, ticket.kick)
+                    && match retirement::stamp_wait(front.observed_at, now_ns(), || {
+                        // A node that cannot be read keeps the bound from the record.
+                        front.work.drained().unwrap_or(true)
+                    }) {
+                        retirement::StampWait::Waiting => false,
+                        retirement::StampWait::Restart(now) => {
+                            restart = Some(now);
+                            false
+                        }
+                        retirement::StampWait::Aged => true,
+                    };
                 let timestamps = retirement::poll(
                     &ticket,
                     front.observed,
@@ -1100,6 +1112,9 @@ impl Queue {
                     }
                 }
                 Ok(None) | Err(EAGAIN) => {
+                    if restart.is_some() {
+                        self.active.iter_mut().next().ok_or(EIO)?.observed_at = restart;
+                    }
                     self.retire_pending = self.front_finished();
                     break;
                 }

drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs

@@ -72,15 +72,16 @@ pub(crate) enum Recordless {
 }
 
 /// How long a command whose exact completion record is present waits for its completion stamp
-/// before it retires on the record alone.
+/// before it retires on the record alone, counted from when its ioctl's work-state node drained.
 ///
 /// The firmware stores the stamp after the cache maintenance that follows the record, so the
 /// stamp, not the record, proves that the command's writes are complete and that its mappings
-/// may be released. While other commands of the same submit ioctl are still running, the stamp
-/// routinely lands several milliseconds after the record (on J700 under games, up to about 3 ms
-/// after the 1 ms this used to wait). Only a stamp still missing well past that is treated as
-/// never stored. The bound stays far below the scheduler timeout, so such a command cannot fail
-/// its VM.
+/// may be released. It stores the stamps of an ioctl's commands only once every kick counted on
+/// the ioctl's work-state node has completed: while other commands of the same submit ioctl
+/// still run, the stamp is not late, however long they run (a command buffer of a dozen render
+/// passes, or a render waiting on another queue). [`stamp_wait`] restarts the bound until the
+/// node drains. Only a stamp still missing well past that is treated as never stored. The bound
+/// stays far below the scheduler timeout, so such a command cannot fail its VM.
 pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000;
 
 // A command that waits out the fallback must still retire well inside the scheduler timeout.
@@ -92,6 +93,33 @@ pub(crate) fn record_aged(observed_at: Option<u64>, now: u64) -> bool {
     observed_at.is_some_and(|at| now.saturating_sub(at) >= STAMP_FALLBACK_NS)
 }
 
+/// Where a command's wait for its completion stamp stands; see [`stamp_wait`].
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum StampWait {
+    /// Within [`STAMP_FALLBACK_NS`] of the record, or no record yet.
+    Waiting,
+    /// The bound passed while the ioctl's work-state node still runs kicks: the stamp is not
+    /// late, and the wait counts again from the given host time.
+    Restart(u64),
+    /// The node drained and the stamp is still missing after the bound: retire on the record.
+    Aged,
+}
+
+/// The stamp wait of a command whose exact record was latched at `observed_at` (host ns) and
+/// whose stamp is still missing, at `now`. `drained` reports whether every kick counted on the
+/// command's work-state node completed; it is asked only once the bound has passed.
+pub(crate) fn stamp_wait(observed_at: Option<u64>, now: u64, drained: impl FnOnce() -> bool)
+    -> StampWait
+{
+    if !record_aged(observed_at, now) {
+        StampWait::Waiting
+    } else if drained() {
+        StampWait::Aged
+    } else {
+        StampWait::Restart(now)
+    }
+}
+
 static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0);
 
 pub(crate) fn stampless_total() -> u64 {
@@ -538,6 +566,22 @@ mod tests {
         assert_eq!(result, Ok(Some([0x1111, 0x2222])));
     }
 
+    #[test]
+    fn a_stamp_waits_for_the_rest_of_its_ioctl() {
+        let at = 1_000_000;
+        let late = at + STAMP_FALLBACK_NS;
+        // The node is asked only once the bound passed.
+        assert_eq!(stamp_wait(Some(at), late - 1, || panic!("node read early")), StampWait::Waiting);
+        assert_eq!(stamp_wait(None, u64::MAX, || panic!("node read without a record")),
+            StampWait::Waiting);
+        // Other kicks of the ioctl still run: the stamp is not late, the bound restarts.
+        assert_eq!(stamp_wait(Some(at), late, || false), StampWait::Restart(late));
+        assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting);
+        // The node drained and the stamp is still missing after the bound.
+        assert_eq!(stamp_wait(Some(late), late + STAMP_FALLBACK_NS, || true), StampWait::Aged);
+        assert_eq!(stamp_wait(Some(at), late, || true), StampWait::Aged);
+    }
+
     #[test]
     fn an_aged_record_still_waits_for_consumed_items() {
         let record = Some(Observation { timestamps: [10, 20] });

drivers/gpu/drm/asahi/g17/queue/render.rs

@@ -1084,7 +1084,30 @@ impl Pair {
             return Ok(false);
         }
         let since = *front.stamp_wait_since.get_or_insert(now);
-        Ok(now.saturating_sub(since) >= retirement::STAMP_FALLBACK_NS)
+        Ok(Self::stamp_wait_expired(front, since, now))
+    }
+
+    /// Whether the front's stamp wait latched at `since` has run out by `now`. While other kicks
+    /// of the group's ioctl still run, its stamps are not late: the wait restarts and the pair
+    /// stays latched for the rescan.
+    fn stamp_wait_expired(front: &mut Active, since: u64, now: u64) -> bool {
+        let drained = || {
+            // A node that cannot be read keeps the bound from the latch.
+            front
+                .packet
+                .completion
+                .work_state()
+                .and_then(|work| work.drained())
+                .unwrap_or(true)
+        };
+        match retirement::stamp_wait(since, now, drained) {
+            retirement::StampWait::Waiting => false,
+            retirement::StampWait::Restart(now) => {
+                front.stamp_wait_since = Some(now);
+                false
+            }
+            retirement::StampWait::Expired => true,
+        }
     }
 
     /// Whether the front group's stamp wait is latched: the observation scan rechecks the pair
@@ -1322,9 +1345,9 @@ impl Pair {
         {
             return Err(EIO);
         }
-        let stampless = front.stamp_wait_since.is_some_and(|since| {
-            crate::g17::now_ns().saturating_sub(since) >= retirement::STAMP_FALLBACK_NS
-        });
+        let stampless = front
+            .stamp_wait_since
+            .is_some_and(|since| Self::stamp_wait_expired(front, since, crate::g17::now_ns()));
         let mut observer = Observer {
             memory: &self.memory,
             qids,

drivers/gpu/drm/asahi/g17/queue/render/retirement.rs

@@ -156,13 +156,17 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result<Observation
 }
 
 /// How long a front group the firmware executed, and whose ring entries it consumed, waits for
-/// its completion stamp before it retires on its pass record.
+/// its completion stamp before it retires on its pass record, counted from when its ioctl's
+/// work-state node drained.
 ///
 /// The firmware stores both queues' stamps after the cache maintenance that follows a pass, so
-/// the stamps, not the pass record, normally prove completion. Occasionally the stamps of a pair
-/// stop one kick short while the pass record shows the group (and later groups) executed, and
-/// no later event moves them. Without a bound the client waits for the scheduler watchdog and
-/// its renewals, then loses its device. Compute applies the same bound to its records.
+/// the stamps, not the pass record, normally prove completion. It stores the stamps of an
+/// ioctl's commands only once every kick counted on the ioctl's work-state node completed: while
+/// other commands of the same ioctl still run, the stamps are not late ([`stamp_wait`]).
+/// Occasionally the stamps of a pair stop one kick short while the pass record shows the group
+/// (and later groups) executed, and no later event moves them. Without a bound the client waits
+/// for the scheduler watchdog and its renewals, then loses its device. Compute applies the same
+/// bound to its records.
 ///
 /// Nothing here proves that the maintenance finished: whether the firmware completes it before
 /// it advances the done cursors is not known. A group retired this way relies on this bound for
@@ -171,6 +175,32 @@ pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result<Observation
 pub(crate) const STAMP_FALLBACK_NS: u64 = 50_000_000;
 const _: () = assert!(STAMP_FALLBACK_NS * 4 <= crate::g17::job::TIMEOUT_MS as u64 * 1_000_000);
 
+/// Where a front group's wait for its stamps stands; see [`stamp_wait`].
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(crate) enum StampWait {
+    /// Within [`STAMP_FALLBACK_NS`] of the latch.
+    Waiting,
+    /// The bound passed while the group's ioctl node still runs kicks: the stamps are not late,
+    /// and the wait counts again from the given host time.
+    Restart(u64),
+    /// The node drained and the stamps still lag after the bound, or the watchdog expired the
+    /// wait (`since` 0): retire on the pass record.
+    Expired,
+}
+
+/// The stamp wait latched at `since` (host ns) at `now`. `drained` reports whether every kick
+/// counted on the group's work-state node completed; it is asked only once the bound has passed
+/// and never for a wait the watchdog expired.
+pub(crate) fn stamp_wait(since: u64, now: u64, drained: impl FnOnce() -> bool) -> StampWait {
+    if now.saturating_sub(since) < STAMP_FALLBACK_NS {
+        StampWait::Waiting
+    } else if since == 0 || drained() {
+        StampWait::Expired
+    } else {
+        StampWait::Restart(now)
+    }
+}
+
 static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0);
 
 /// Counts a render group that retired on its pass record because its completion stamps never
@@ -484,4 +514,20 @@ mod executed_tests {
         let mut memory = stalled([10, 0]);
         assert!(matches!(poll_stampless(&mut ticket, &mut memory), Ok(Poll::Pending)));
     }
+
+    #[test]
+    fn stamps_wait_for_the_rest_of_the_ioctl() {
+        let since = 1_000_000;
+        let late = since + STAMP_FALLBACK_NS;
+        // The node is asked only once the bound passed.
+        assert_eq!(stamp_wait(since, late - 1, || panic!("node read early")), StampWait::Waiting);
+        // Other kicks of the group's ioctl still run: the bound restarts.
+        assert_eq!(stamp_wait(since, late, || false), StampWait::Restart(late));
+        assert_eq!(stamp_wait(late, late + STAMP_FALLBACK_NS - 1, || true), StampWait::Waiting);
+        // A drained node with lagging stamps is the stall the bound exists for.
+        assert_eq!(stamp_wait(since, late, || true), StampWait::Expired);
+        // The watchdog's last resort does not wait for the node.
+        assert_eq!(stamp_wait(0, STAMP_FALLBACK_NS, || panic!("node read for an expired wait")),
+            StampWait::Expired);
+    }
 }