diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index e06fcded9047..387aaeca1b6c 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -24,10 +24,9 @@ impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { use crate::m3::m3_runtime::Service; - // Snapshots after a notification before sleeping for the next one: the completion - // stamps can trail the firmware's event message. - const TRAILING_POLLS: u32 = 4; - let (mut trailing, mut waited_at) = (0, u64::MAX); + // No deliberate spin: fsleep(5) selects udelay, not a scheduler sleep. + // Check immediately on IRQ, with backoff for trailing coherent witnesses. + let mut policy = super::m3_completion_policy::WaitPolicy::new(); loop { let (outcome, events) = { let mut guard = owner.shared.lock(); @@ -37,23 +36,20 @@ impl WorkItem for Completion { // Preserve the root-only CPU-overlap diagnostic: disabled waits retain // the runtime lock so VM/timestamp preparation cannot overlap the GPU. if let Service::Waiting(messages)=outcome { - if !events.cpu_overlap() {events.wait_past(messages);continue;} + if !events.cpu_overlap() { + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); + continue; + } } (outcome,events) }; match outcome { Service::Idle => break, - Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Progress => policy.progress(), Service::Waiting(messages) => { - if messages != waited_at { waited_at = messages; trailing = 0; } - if trailing < TRAILING_POLLS { - trailing += 1; - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); - } else { - // Until the next firmware event, or a bounded 100 us resnapshot. - events.wait_past(messages); - trailing = 0; - } + // IRQs and timer expiry alone are not retirement progress. + // Keep backing off until service publishes or retires work. + events.wait_past(messages, policy.next_wait_us(events.wait_ceiling_us())); } } } diff --git a/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs new file mode 100644 index 000000000000..9c0da8c3246e --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion_policy.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded fallback for witnesses which can become visible after the last IRQ. +//! +//! Notifications request a fresh snapshot, but do not prove witness progress. +//! Only publication or retirement resets this policy. In particular, neither a +//! timer wake nor a stream of early firmware notifications renews a spin budget. + +pub(crate) struct WaitPolicy { + next_us: u64, +} + +impl WaitPolicy { + pub(crate) const fn new() -> Self { + Self { next_us: 25 } + } + + pub(crate) fn progress(&mut self) { + self.next_us = 25; + } + + /// No deliberate busy delay: IRQs still wake each sleep immediately. + /// The caller supplies the root-selected ceiling (25..=1000 us). + pub(crate) fn next_wait_us(&mut self, ceiling_us: u64) -> u64 { + let wait = self.next_us.min(ceiling_us.clamp(25, 1000)); + self.next_us = (wait * 2).min(1000); + wait + } +} + +#[cfg_attr(not(test), kernel::macros::kunit_tests(asahi_m3_completion_policy))] +mod tests { + use super::WaitPolicy; + + #[test] + fn no_progress_backs_off_and_stays_bounded() { + let mut policy = WaitPolicy::new(); + for expected in [25, 50, 100, 200, 400, 800, 1000, 1000] { + assert_eq!(policy.next_wait_us(1000), expected); + } + for _ in 0..100_000 { + assert_eq!(policy.next_wait_us(1000), 1000); + } + } + + #[test] + fn publication_or_retirement_restarts_fast_detection() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + policy.progress(); + assert_eq!(policy.next_wait_us(1000), 25); + assert_eq!(policy.next_wait_us(1000), 50); + } + + #[test] + fn live_ceiling_changes_apply_to_the_next_wait() { + let mut policy = WaitPolicy::new(); + for _ in 0..8 { policy.next_wait_us(1000); } + assert_eq!(policy.next_wait_us(25), 25); + assert_eq!(policy.next_wait_us(1000), 50); + assert_eq!(policy.next_wait_us(100), 100); + assert_eq!(policy.next_wait_us(100), 100); + } + + #[test] + fn fallback_keeps_timeout_checks_frequent_without_any_irq() { + let mut policy = WaitPolicy::new(); + let mut elapsed_us = 0; + while elapsed_us < 2_000_000 { + let wait = policy.next_wait_us(1000); + assert!(wait <= 1000); + elapsed_us += wait; + } + // This bounds the requested delay, not scheduler dispatch latency. + assert!(elapsed_us < 2_001_000); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index a8ff85a6f5f1..b11b612184ed 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -38,7 +38,7 @@ struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); - writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + writeln!(f, "version=2 policy=adaptive wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", us, irqs, sleeps, timers, preparations) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index be81c4a4e9d9..ba8ad365ce20 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -38,7 +38,7 @@ impl EventWait { Arc::pin_init( pin_init!(EventWait { count: AtomicU64::new(0), - wait_us: AtomicU64::new(100), + wait_us: AtomicU64::new(1000), overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), @@ -55,7 +55,7 @@ impl EventWait { self.arrived.complete(); } - /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// Diagnostic tuning sets the adaptive fallback ceiling in microseconds, never /// stamps, pipe retirement, event counts, fences or the two-second timeout. /// Atomic publication permits root to compare policies in the same boot. pub(crate) fn set_wait_us(&self, us: u64) -> Result { @@ -64,6 +64,10 @@ impl EventWait { Ok(()) } + pub(crate) fn wait_ceiling_us(&self) -> u64 { + self.wait_us.load(Ordering::Relaxed) + } + pub(crate) fn cpu_overlap(&self) -> bool { self.overlap_cpu.load(Ordering::Relaxed) } @@ -82,11 +86,11 @@ impl EventWait { } /// Sleep until a notification later than `seen` has been recorded or - /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// the selected 25..1000 us fallback elapses. A one-jiffy fallback alone takes up to 4 ms at HZ=250, /// even when the GPU stamp arrives just after its notification. The timer /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. /// Only the single completion worker may call this method. - pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + pub(crate) fn wait_past(self: &Arc, seen: u64, wait_us: u64) -> bool { // Events can arrive while the worker is polling. Discard stale wakeup // tokens before checking the sequence. A concurrent event either moves // the sequence or leaves a token, including between the check and wait. @@ -98,7 +102,7 @@ impl EventWait { // on every return, before a subsequent wait can arm the timer. self.sleeps.fetch_add(1, Ordering::Relaxed); let timer = self.clone().start(Delta::from_micros( - self.wait_us.load(Ordering::Relaxed) as i64)); + wait_us.clamp(25, 1000) as i64)); // Retain one tick as a backup bound; normally either the real // interrupt or our high-resolution timer supplies the token. self.arrived.wait_for_completion_timeout(1); diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 082449e34cf4..e34236fd57a3 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -1079,6 +1079,8 @@ impl Runtime { crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { + // Without module unloading, the C helper is an unbound no-op. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe {kernel::bindings::__module_get(kernel::module::this_module::().as_ptr())}; core::mem::forget(transport);core::mem::forget(state); core::mem::forget(adopted); @@ -1348,6 +1350,7 @@ impl Drop for Runtime { for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { // SAFETY: a live module reference pins callbacks with retained DMA. + #[cfg(CONFIG_MODULE_UNLOAD)] unsafe { kernel::bindings::__module_get(kernel::module::this_module::().as_ptr()) }; return; } diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index d885da1a8647..eafd5732bcea 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -133,6 +133,8 @@ mod t8122_knobs; #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] mod t8122_start; +mod m3_completion_policy; + mod m3_compute_layout; mod m3_compute_sequence; diff --git a/tools/asahi/m3-fence-latency.py b/tools/asahi/m3-fence-latency.py new file mode 100644 index 000000000000..c8428d65dbf0 --- /dev/null +++ b/tools/asahi/m3-fence-latency.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Join existing M3 timing logs. Input must be one boot, captured without gaps.""" +import collections +import json +import math +import re +import sys + +milestones = {} +packets = collections.defaultdict(list) +fences = [] +invalid = collections.Counter() +for line in open(sys.argv[1]): + fields = {k: int(v, 0) for k, v in re.findall(r'(\w+)=(-?0x[0-9a-fA-F]+|-?\d+)', line)} + if 'M3_BATCH_MILESTONE ' in line: + key = (fields['vm'], fields['publication_tick']) + if key in milestones: + invalid['duplicate_milestone'] += 1 + milestones[key] = fields + elif 'M3_BATCH_PACKET ' in line: + packets[(fields['vm'], fields['packet'])].append((fields['publication_tick'], fields['command'])) + elif 'M3_PACKET_FENCE ' in line: + fences.append(fields) +values = [] +previous_signal = {} +for f in sorted(fences, key=lambda f: f['signal_before_tick']): + identity = (f['vm'], f['packet']) + previous = previous_signal.get(identity, 0) + previous_signal[identity] = f['signal_after_tick'] + if f['status'] != 0: + invalid['error_fence'] += 1 + continue + linked = packets.get((f['vm'], f['packet']), []) + # Pointer reuse is disambiguated by the completed fence's physical counter. + candidates = [(tick, command) for tick, command in linked if previous < tick <= f['signal_before_tick'] and command == f['commands'] - 1] + if not candidates: + invalid['missing_final_command'] += 1 + continue + final_tick, _ = max(candidates) + batch = milestones.get((f['vm'], final_tick)) + if not batch or batch['mixed'] or batch['render_end_tick'] <= 0: + invalid['missing_or_non_render_end'] += 1 + continue + end = batch['render_end_tick'] + before, after = f['signal_before_tick'], f['signal_after_tick'] + if not (final_tick <= end <= before <= after): + invalid['invalid_counter_order'] += 1 + continue + # GPU execution end is earlier than full coherent retirement; this includes + # trailing witnesses and dispatch/retirement, rather than claiming an IRQ is done. + values.append(((before - end) / 24, (after - end) / 24)) + +def percentiles(data): + ordered = sorted(data) + return {str(p): ordered[max(0, math.ceil(len(ordered) * p / 100) - 1)] for p in (50, 95, 99, 100)} if ordered else {} + +print(json.dumps({'render_fences': len(values), 'invalid': invalid, + 'gpu_end_to_signal_before_us': percentiles([a for a, b in values]), + 'gpu_end_to_signal_after_us': percentiles([b for a, b in values]), + 'note': '24 MHz physical counter; interval brackets signal(); includes trailing witnesses. Not presentation latency.'}, indent=2)) +if not values: + sys.exit('No valid render fences: do not report latency from this capture') diff --git a/tools/asahi/m3-runtime-tests.rs b/tools/asahi/m3-runtime-tests.rs index 91cbefd298fe..0d6d4e584119 100644 --- a/tools/asahi/m3-runtime-tests.rs +++ b/tools/asahi/m3-runtime-tests.rs @@ -106,3 +106,6 @@ fn extended_parameter_aliases_are_gpu_only_and_legacy_is_unchanged() { } } } + +#[path = "../../drivers/gpu/drm/asahi/m3/m3_completion_policy.rs"] +mod m3_completion_policy;