drm/asahi: g17: admit a context's ioctls to work states in submission order (stacked on #275)

drivers/gpu/drm/asahi/g17/context.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Logical queue ownership of scheduler, QoS and work-state identities.
@@ -15,7 +15,7 @@
 };
 use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ};
 use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU8, AtomicUsize, Ordering};
-use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT};
+use identity::{Order, RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT};
 use super::job::Engine;
 use kernel::{
     new_mutex,
@@ -128,6 +128,11 @@
     states: WorkStates,
     /// Signalled when a work state returns; publishers that found none wait on it.
     state_returns: Arc<super::availability::Availability>,
+    /// The domain refused compute publishers wait on; signalled with `state_returns` when the
+    /// admission turn moves on.
+    compute_returns: Arc<super::availability::Availability>,
+    /// The order in which this context's ioctls take work-state nodes.
+    order: Arc<Mutex<Order>>,
     work: Arc<Mutex<WorkStorage>>,
     work_base: u64,
     render: Arc<Mutex<Option<Arc<RenderPool>>>>,
@@ -139,6 +144,10 @@
     policy: Policy,
 }
 impl Context {
+    #[expect(
+        clippy::too_many_arguments,
+        reason = "A context takes its allocators, identities and both admission domains."
+    )]
     pub(crate) fn new(
         alloc: &Allocator<'_>,
         pool: &Arc<Pool>,
@@ -147,6 +156,7 @@
         owner_pid: u32,
         policy: Policy,
         state_returns: Arc<super::availability::Availability>,
+        compute_returns: Arc<super::availability::Availability>,
     ) -> Result<Self> {
         let status = execution.vm().status().ok_or(EINVAL)?.clone();
         let qos = qos.acquire()?;
@@ -195,6 +205,8 @@
             GFP_KERNEL,
         )?;
         let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?;
+        let order =
+            Arc::pin_init(new_mutex!(Order::new(), "G17 context admission order"), GFP_KERNEL)?;
         Ok(Self {
             execution,
             status,
@@ -208,6 +220,8 @@
             owner_pid,
             states: WorkStates::new(),
             state_returns,
+            compute_returns,
+            order,
             work,
             work_base,
             render,
@@ -399,18 +413,23 @@
     word: AtomicU32,
     /// Index of the engine whose admission took the node, valid once `word` is set.
     engine: AtomicU8,
+    /// The ioctl's place in its context's submission order.
+    ordinal: u64,
 }
 impl WorkStateLease {
     pub(super) fn status(&self) -> &Arc<VmStatus> { self.context.status() }
     pub(crate) fn deferred(context: Arc<Context>) -> Self {
+        let ordinal = context.order.lock().assign();
         Self {
             context,
             word: AtomicU32::new(FREE),
             engine: AtomicU8::new(0),
+            ordinal,
         }
     }
-    /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: none
-    /// may be taken now (see `identity::may_take`); a later return makes one available.
+    /// The ioctl's node, taking one for `engine`'s admission if it has none yet. EBUSY: too many
+    /// older ioctls still wait for theirs (see `identity::Order`), or no node may be taken now
+    /// (see `identity::may_take`); an older ioctl's admission or a return lets it try again.
     pub(crate) fn acquire(&self, engine: Engine) -> Result<u32> {
         let current = self.word();
         if current != FREE {
@@ -423,6 +442,10 @@
         if current != FREE {
             return Ok(current);
         }
+        let mut order = self.context.order.lock();
+        if !order.admits(self.ordinal) {
+            return Err(EBUSY);
+        }
         let engine = engine.index();
         let candidate = self.context.states.allocate(engine).ok_or(EBUSY)?;
         if let Err(error) = work.initialize(candidate) {
@@ -431,6 +454,12 @@
         }
         self.engine.store(engine as u8, Ordering::Relaxed);
         self.word.store(candidate, Ordering::Release);
+        let wake = order.admit(self.ordinal);
+        drop(order);
+        drop(work);
+        if wake {
+            self.context.pass_turn();
+        }
         Ok(candidate)
     }
     /// The domain a publisher samples before acquire() and waits on after EBUSY.
@@ -478,6 +507,8 @@
         let word = self.word();
         if word != FREE {
             self.context.release_state(word, usize::from(self.engine.load(Ordering::Relaxed)));
+        } else if self.context.order.lock().leave(self.ordinal) {
+            self.context.pass_turn();
         }
     }
 }
@@ -490,6 +521,12 @@
             self.state_returns.notify();
         }
     }
+    /// A refused publisher waits on its engine's domain. Bumping both once the turn moved past a
+    /// refusal means one that sampled before its refused acquire() never sleeps through it.
+    fn pass_turn(&self) {
+        self.state_returns.notify();
+        self.compute_returns.notify();
+    }
 }
 
 impl QosIds {

drivers/gpu/drm/asahi/g17/context/identity.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Context-local work-state identities. The low six bits address a node;
@@ -77,6 +77,101 @@
     }
 }
 
+/// Ordinals past the turn remembered for ioctls that took a node or left early.
+pub(super) const ORDER_WINDOW: u64 = 1024;
+
+/// How far past the oldest waiting ioctl an ioctl may take its node.
+pub(super) const RUN_AHEAD: u64 = 8;
+
+/// The order in which a context's ioctls take work-state nodes.
+///
+/// Each engine admits a context's ioctls in submission order, but the engines advance
+/// independently: render admission can take nodes for ioctls far ahead of an older ioctl still
+/// waiting for its turn on the compute queue. Under deep mixed load (128 compute and 64 render
+/// ioctls in flight on one queue) the firmware then stops completing all of the context's work
+/// within seconds. Native macOS schedules a queue's command buffers in commit order across
+/// engines, at most 36 between scheduling and completion, and the same load runs there without
+/// failure.
+///
+/// So an ioctl takes its node only while at most [`RUN_AHEAD`] older ioctls of its context have
+/// neither taken one nor left without one. Native scheduling happens at commit, on one thread;
+/// here each engine admits from its own scheduler, and a strictly ordered turn costs a scheduler
+/// round trip whenever the other engine's ioctl is ready first. A short run-ahead avoids that
+/// without letting one engine get far ahead of the other.
+pub(super) struct Order {
+    /// The next ordinal to hand out.
+    next: u64,
+    /// The oldest ordinal that has neither taken a node nor left.
+    turn: u64,
+    /// Ordinals past the turn that took a node or left, as bits at ordinal % ORDER_WINDOW.
+    done: [u64; (ORDER_WINDOW / 64) as usize],
+    /// Set once an ioctl left too far past the turn to remember; admission is unordered after.
+    unordered: bool,
+    /// An ioctl was refused since the turn last moved: its publisher waits for the next move.
+    refused: bool,
+}
+impl Order {
+    pub(super) const fn new() -> Self {
+        Self {
+            next: 0,
+            turn: 0,
+            done: [0; (ORDER_WINDOW / 64) as usize],
+            unordered: false,
+            refused: false,
+        }
+    }
+    /// The ordinal of a new ioctl, in submission order.
+    pub(super) fn assign(&mut self) -> u64 {
+        let ordinal = self.next;
+        self.next += 1;
+        ordinal
+    }
+    /// Whether the ioctl `ordinal` may take its node now. A refusal is remembered until the
+    /// turn next moves, so that move wakes the refused publisher.
+    pub(super) fn admits(&mut self, ordinal: u64) -> bool {
+        let admits = self.unordered || ordinal <= self.turn + RUN_AHEAD;
+        self.refused |= !admits;
+        admits
+    }
+    /// The ioctl `ordinal` took its node. Returns whether a refused ioctl may now be admitted.
+    pub(super) fn admit(&mut self, ordinal: u64) -> bool {
+        self.finish(ordinal)
+    }
+    /// The ioctl `ordinal` left without taking a node (it failed or was cancelled first).
+    /// Returns whether a refused ioctl may now be admitted.
+    pub(super) fn leave(&mut self, ordinal: u64) -> bool {
+        self.finish(ordinal)
+    }
+    fn finish(&mut self, ordinal: u64) -> bool {
+        if self.unordered || ordinal < self.turn {
+            return false;
+        }
+        if ordinal == self.turn {
+            self.pass();
+            return core::mem::take(&mut self.refused);
+        }
+        if ordinal - self.turn >= ORDER_WINDOW {
+            self.unordered = true;
+            return core::mem::take(&mut self.refused);
+        }
+        let bit = ordinal % ORDER_WINDOW;
+        self.done[(bit / 64) as usize] |= 1 << (bit % 64);
+        false
+    }
+    /// Moves the turn past the current ordinal and every following one already done.
+    fn pass(&mut self) {
+        loop {
+            self.turn += 1;
+            let bit = self.turn % ORDER_WINDOW;
+            let (word, mask) = ((bit / 64) as usize, 1u64 << (bit % 64));
+            if self.done[word] & mask == 0 {
+                break;
+            }
+            self.done[word] &= !mask;
+        }
+    }
+}
+
 /// Two stable logical ordinals, independent of physical pair numbering.
 pub(super) struct RenderSlots {
     pairs: [Option<u8>; 2],
@@ -154,6 +249,71 @@
         assert!(states.allocate(1).is_some());
     }
 
+    #[test]
+    fn ioctls_take_nodes_at_most_run_ahead_past_the_oldest_waiting_one() {
+        let mut order = Order::new();
+        let ordinals: [u64; RUN_AHEAD as usize + 3] = core::array::from_fn(|_| order.assign());
+        let (oldest, last) = (ordinals[0], ordinals[RUN_AHEAD as usize + 2]);
+        // Every ioctl up to RUN_AHEAD past the oldest waiting one may take its node.
+        for &ordinal in &ordinals[..=RUN_AHEAD as usize] {
+            assert!(order.admits(ordinal));
+        }
+        assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1]));
+        // Ioctls ahead of the turn taking their nodes do not move it.
+        for &ordinal in &ordinals[1..=RUN_AHEAD as usize] {
+            assert!(!order.admit(ordinal));
+        }
+        assert!(!order.admits(ordinals[RUN_AHEAD as usize + 1]));
+        // The oldest one moves the turn past everything already admitted, and wakes the refused.
+        assert!(order.admit(oldest));
+        assert!(order.admits(last));
+        let next = order.assign();
+        assert!(order.admits(next));
+        assert!(!order.admits(next + RUN_AHEAD + 1));
+    }
+
+    #[test]
+    fn ioctls_leaving_without_a_node_pass_the_turn() {
+        let mut order = Order::new();
+        let (a, b, c) = (order.assign(), order.assign(), order.assign());
+        let later: [u64; RUN_AHEAD as usize] = core::array::from_fn(|_| order.assign());
+        let beyond = later[RUN_AHEAD as usize - 1];
+        // b and c fail before their turn; while a waits, ioctls past a + RUN_AHEAD wait too.
+        assert!(!order.leave(b));
+        assert!(!order.leave(c));
+        assert!(!order.admits(beyond));
+        // Once a takes its node, the turn passes b and c as well, and the refused one may retry.
+        assert!(order.admit(a));
+        assert!(order.admits(beyond));
+        // With nobody refused since, moving the turn wakes nobody; an ordinal already passed
+        // leaves without effect.
+        assert!(!order.leave(later[0]));
+        assert!(!order.leave(a));
+    }
+
+    #[test]
+    fn an_ioctl_leaving_too_far_ahead_unorders_admission() {
+        let mut order = Order::new();
+        let first = order.assign();
+        let mut last = first;
+        for _ in 0..ORDER_WINDOW {
+            last = order.assign();
+        }
+        assert!(!order.admits(last));
+        assert!(order.leave(last));
+        assert!(order.admits(last - 1));
+    }
+
+    #[test]
+    fn single_engine_admission_wakes_nobody() {
+        let mut order = Order::new();
+        for _ in 0..3 * RUN_AHEAD {
+            let ordinal = order.assign();
+            assert!(order.admits(ordinal));
+            assert!(!order.admit(ordinal));
+        }
+    }
+
     #[test]
     fn releases_return_nodes_to_their_engine() {
         let states = WorkStates::new();

drivers/gpu/drm/asahi/g17/runtime.rs

@@ -1,3 +1,3 @@
 // SPDX-License-Identifier: GPL-2.0-only OR MIT
 
 //! Device ownership of physical queues and off-lock construction reservations.
@@ -754,7 +754,16 @@
         let owner_pid =
             u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?;
         let context = Arc::new(
-            Context::new(&alloc, &pool, &qos, execution, owner_pid, policy, shared.renders.clone())?,
+            Context::new(
+                &alloc,
+                &pool,
+                &qos,
+                execution,
+                owner_pid,
+                policy,
+                shared.renders.clone(),
+                shared.progress.clone(),
+            )?,
             GFP_KERNEL,
         )?;
         let state = shared.state.lock();