drm/asahi: back off interrupt-assisted M3 completion waits

drivers/gpu/drm/asahi/m3/m3_completion.rs

@@ -24,10 +24,9 @@ impl WorkItem for Completion {
     type Pointer = Arc<Self>;
     fn run(owner: Arc<Self>) {
         use crate::m3::m3_runtime::Service;
-        // Snapshots after a notification before sleeping for the next one: the completion
-        // stamps can trail the firmware's event message.
-        const TRAILING_POLLS: u32 = 4;
-        let (mut trailing, mut waited_at) = (0, u64::MAX);
+        // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep.
+        // Check immediately on IRQ, with backoff for trailing coherent witnesses.
+        let mut policy = super::m3_completion_policy::WaitPolicy::new();
         loop {
             let (outcome, events) = {
                 let mut guard = owner.shared.lock();
@@ -37,23 +36,20 @@ impl WorkItem for Completion {
                 // Preserve the root-only CPU-overlap diagnostic: disabled waits retain
                 // the runtime lock so VM/timestamp preparation cannot overlap the GPU.
                 if let Service::Waiting(messages)=outcome {
-                    if !events.cpu_overlap() {events.wait_past(messages);continue;}
+                    if !events.cpu_overlap() {
+                        events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us()));
+                        continue;
+                    }
                 }
                 (outcome,events)
             };
             match outcome {
                 Service::Idle => break,
-                Service::Progress => { trailing = 0; waited_at = u64::MAX; }
+                Service::Progress => policy.progress(),
                 Service::Waiting(messages) => {
-                    if messages != waited_at { waited_at = messages; trailing = 0; }
-                    if trailing < TRAILING_POLLS {
-                        trailing += 1;
-                        kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5));
-                    } else {
-                        // Until the next firmware event, or a bounded 100 us resnapshot.
-                        events.wait_past(messages);
-                        trailing = 0;
-                    }
+                    // IRQs and timer expiry alone are not retirement progress.
+                    // Keep backing off until service publishes or retires work.
+                    events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us()));
                 }
             }
         }

drivers/gpu/drm/asahi/m3/m3_completion_policy.rs

@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Bounded fallback for witnesses which can become visible after the last IRQ.
+//!
+//! Notifications request a fresh snapshot, but do not prove witness progress.
+//! Only publication or retirement resets this policy. In particular, neither a
+//! timer wake nor a stream of early firmware notifications renews a spin budget.
+
+pub(crate) struct WaitPolicy {
+    next_us: u64,
+}
+
+impl WaitPolicy {
+    pub(crate) const fn new() -> Self {
+        Self { next_us: 25 }
+    }
+
+    pub(crate) fn progress(&mut self) {
+        self.next_us = 25;
+    }
+
+    /// No deliberate busy delay: IRQs still wake each sleep immediately.
+    /// The caller supplies the root-selected ceiling (25..=1000 us).
+    pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 {
+        let wait = self.next_us.min(ceiling_us.clamp(25, 1000));
+        self.next_us = (wait * 2).min(1000);
+        wait
+    }
+}
+
+#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))]
+mod tests {
+    use super::WaitPolicy;
+
+    #[test]
+    fn no_progress_backs_off_and_stays_bounded() {
+        let mut policy = WaitPolicy::new();
+        for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] {
+            assert_eq!(policy.next_wait_us(1000), expected);
+        }
+        for _ in 0..100_000 {
+            assert_eq!(policy.next_wait_us(1000), 1000);
+        }
+    }
+
+    #[test]
+    fn publication_or_retirement_restarts_fast_detection() {
+        let mut policy = WaitPolicy::new();
+        for _ in 0..8 { policy.next_wait_us(1000); }
+        policy.progress();
+        assert_eq!(policy.next_wait_us(1000), 25);
+        assert_eq!(policy.next_wait_us(1000), 50);
+    }
+
+    #[test]
+    fn live_ceiling_changes_apply_to_the_next_wait() {
+        let mut policy = WaitPolicy::new();
+        for _ in 0..8 { policy.next_wait_us(1000); }
+        assert_eq!(policy.next_wait_us(25), 25);
+        assert_eq!(policy.next_wait_us(1000), 50);
+        assert_eq!(policy.next_wait_us(100), 100);
+        assert_eq!(policy.next_wait_us(100), 100);
+    }
+
+    #[test]
+    fn fallback_keeps_timeout_checks_frequent_without_any_irq() {
+        let mut policy = WaitPolicy::new();
+        let mut elapsed_us = 0;
+        while elapsed_us < 2_000_000 {
+            let wait = policy.next_wait_us(1000);
+            assert!(wait <= 1000);
+            elapsed_us += wait;
+        }
+        // This bounds the requested delay, not scheduler dispatch latency.
+        assert!(elapsed_us < 2_001_000);
+    }
+}

drivers/gpu/drm/asahi/m3/m3_drm.rs

@@ -38,7 +38,7 @@ struct CompletionView(Arc<crate::m3::m3_rtkit::EventWait>);
 impl kernel::debugfs::Writer for CompletionView {
     fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
         let (us, irqs, sleeps, timers, preparations) = self.0.snapshot();
-        writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}",
+        writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}",
                  us, irqs, sleeps, timers, preparations)
     }
 }

drivers/gpu/drm/asahi/m3/m3_rtkit.rs

@@ -38,7 +38,7 @@ impl EventWait {
         Arc::pin_init(
             pin_init!(EventWait {
                 count: AtomicU64::new(0),
-                wait_us: AtomicU64::new(100),
+                wait_us: AtomicU64::new(1000),
                 overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()),
                 cpu_preparations: AtomicU64::new(0),
                 sleeps: AtomicU64::new(0),
@@ -55,7 +55,7 @@ impl EventWait {
         self.arrived.complete();
     }
 
-    /// Diagnostic tuning changes only the bounded resnapshot cadence, never
+    /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never
     /// stamps, pipe retirement, event counts, fences or the two-second timeout.
     /// Atomic publication permits root to compare policies in the same boot.
     pub(crate) fn set_wait_us(&self, us: u64) -> Result {
@@ -64,6 +64,10 @@ impl EventWait {
         Ok(())
     }
 
+    pub(crate) fn wait_ceiling_us(&self) -> u64 {
+        self.wait_us.load(Ordering::Relaxed)
+    }
+
     pub(crate) fn cpu_overlap(&self) -> bool {
         self.overlap_cpu.load(Ordering::Relaxed)
     }
@@ -82,11 +86,11 @@ impl EventWait {
     }
 
     /// Sleep until a notification later than `seen` has been recorded or
-    /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250,
+    /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250,
     /// even when the GPU stamp arrives just after its notification. The timer
     /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting.
     /// Only the single completion worker may call this method.
-    pub(crate) fn wait_past(self: &Arc<Self>, seen: u64) -> bool {
+    pub(crate) fn wait_past(self: &Arc<Self>, seen: u64, wait_us: u64) -> bool {
         // Events can arrive while the worker is polling. Discard stale wakeup
         // tokens before checking the sequence. A concurrent event either moves
         // the sequence or leaves a token, including between the check and wait.
@@ -98,7 +102,7 @@ impl EventWait {
                 // on every return, before a subsequent wait can arm the timer.
                 self.sleeps.fetch_add(1, Ordering::Relaxed);
                 let timer = self.clone().start(Delta::from_micros(
-                    self.wait_us.load(Ordering::Relaxed) as i64));
+                    wait_us.clamp(25, 1000) as i64));
                 // Retain one tick as a backup bound; normally either the real
                 // interrupt or our high-resolution timer supplies the token.
                 self.arrived.wait_for_completion_timeout(1);

drivers/gpu/drm/asahi/m3/m3_runtime.rs

@@ -1079,6 +1079,8 @@ impl Runtime {
                 crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e);
                 state.health.mark_failed();
                 if device.stop_asc().is_err() {
+                    // Without module unloading, the C helper is an unbound no-op.
+                    #[cfg(CONFIG_MODULE_UNLOAD)]
                     unsafe {kernel::bindings::__module_get(kernel::module::this_module::<crate::LocalModule>().as_ptr())};
                     core::mem::forget(transport);core::mem::forget(state);
                     core::mem::forget(adopted);
@@ -1348,6 +1350,7 @@ impl Drop for Runtime {
         for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));}
         if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) {
             // SAFETY: a live module reference pins callbacks with retained DMA.
+            #[cfg(CONFIG_MODULE_UNLOAD)]
             unsafe { kernel::bindings::__module_get(kernel::module::this_module::<crate::LocalModule>().as_ptr()) };
             return;
         }

drivers/gpu/drm/asahi/m3/mod.rs

@@ -133,6 +133,8 @@ mod t8122_knobs;
 #[cfg(CONFIG_DRM_ASAHI_M3_T8122)]
 mod t8122_start;
 
+mod m3_completion_policy;
+
 mod m3_compute_layout;
 
 mod m3_compute_sequence;

tools/asahi/m3-fence-latency.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only OR MIT
+"""Join existing M3 timing logs. Input must be one boot, captured without gaps."""
+import collections
+import json
+import math
+import re
+import sys
+
+milestones = {}
+packets = collections.defaultdict(list)
+fences = []
+invalid = collections.Counter()
+for line in open(sys.argv[1]):
+    fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)}
+    if 'M3_BATCH_MILESTONE ' in line:
+        key = (fields['vm'], fields['publication_tick'])
+        if key in milestones:
+            invalid['duplicate_milestone'] += 1
+        milestones[key] = fields
+    elif 'M3_BATCH_PACKET ' in line:
+        packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command']))
+    elif 'M3_PACKET_FENCE ' in line:
+        fences.append(fields)
+values = []
+previous_signal = {}
+for f in sorted(fences, key=lambda f: f['signal_before_tick']):
+    identity = (f['vm'], f['packet'])
+    previous = previous_signal.get(identity, 0)
+    previous_signal[identity] = f['signal_after_tick']
+    if f['status'] != 0:
+        invalid['error_fence'] += 1
+        continue
+    linked = packets.get((f['vm'], f['packet']), [])
+    # Pointer reuse is disambiguated by the completed fence's physical counter.
+    candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1]
+    if not candidates:
+        invalid['missing_final_command'] += 1
+        continue
+    final_tick, _ = max(candidates)
+    batch = milestones.get((f['vm'], final_tick))
+    if not batch or batch['mixed'] or batch['render_end_tick'] <= 0:
+        invalid['missing_or_non_render_end'] += 1
+        continue
+    end = batch['render_end_tick']
+    before, after = f['signal_before_tick'], f['signal_after_tick']
+    if not (final_tick <= end <= before <= after):
+        invalid['invalid_counter_order'] += 1
+        continue
+    # GPU execution end is earlier than full coherent retirement; this includes
+    # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done.
+    values.append(((before - end) / 24, (after - end) / 24))
+
+def percentiles(data):
+    ordered = sorted(data)
+    return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {}
+
+print(json.dumps({'render_fences': len(values), 'invalid': invalid,
+                  'gpu_end_to_signal_before_us': percentiles([a for a, b in values]),
+                  'gpu_end_to_signal_after_us': percentiles([b for a, b in values]),
+                  'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2))
+if not values:
+    sys.exit('No valid render fences: do not report latency from this capture')

tools/asahi/m3-runtime-tests.rs

@@ -106,3 +106,6 @@ fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() {
         }
     }
 }
+
+#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"]
+mod m3_completion_policy;