From e51f1082fa2338c8119b9a64ec7f17cf659cf862 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Thu, 22 Oct 2026 23:16:35 -0400 Subject: [PATCH 1/8] drm/asahi: m3: restore the 7.1.12 T8122 runtime sources, unwired Bring back, unchanged, the M3 runtime that starts the T8122 (M3 MacBook Air) GPU on the release-20 kernel (c46d7dcdb871, the source of linux-aurora-m3gpu 7.1.12.aurora2-20261017.20): the runtime, its firmware admission, InitData, device, RTKit, memory, submission and client files, the T8122 start experiment, and the 7.1.12 hw tables (hw.rs, hw/t8122.rs, hw/agx3.rs, identity.rs). g15_raw.rs and initdata.rs are the G15 (G15, V14_8_3) expansions of 7.1.12's fw/initdata.rs and initdata.rs: every #[versions(AGX)] item replaced by its G15V14_8_3 instance, which 7.2's version matrix no longer generates. The expansion keeps the source text; its eleven G15 structures match rustc's own expansion of c46d7dcdb871 field for field. Nothing declares these modules yet, so this changes no build. The next commits adapt them to the 7.2 driver. Co-authored-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-authored-by: Eryk Wieliczko Co-authored-by: Satya Benson Co-authored-by: Naeem Malik Co-authored-by: garymeyerca Co-authored-by: DjDeveloperr Generated-by: LLM Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/agx_attachments.rs | 71 + drivers/gpu/drm/asahi/m3/agx_compute.rs | 9 + drivers/gpu/drm/asahi/m3/agx_fault.rs | 87 + drivers/gpu/drm/asahi/m3/agx_memory.rs | 12 + drivers/gpu/drm/asahi/m3/agx_resources.rs | 170 ++ drivers/gpu/drm/asahi/m3/agx_status.rs | 34 + drivers/gpu/drm/asahi/m3/agx_uapi.rs | 721 ++++++++ drivers/gpu/drm/asahi/m3/g15_raw.rs | 1800 +++++++++++++++++++ drivers/gpu/drm/asahi/m3/hw.rs | 679 +++++++ drivers/gpu/drm/asahi/m3/hw/agx3.rs | 59 + drivers/gpu/drm/asahi/m3/hw/t8122.rs | 147 ++ drivers/gpu/drm/asahi/m3/identity.rs | 67 + drivers/gpu/drm/asahi/m3/initdata.rs | 1457 +++++++++++++++ drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 1491 +++++++++++++++ drivers/gpu/drm/asahi/m3/m3_board.rs | 559 ++++++ drivers/gpu/drm/asahi/m3/m3_client.rs | 228 +++ drivers/gpu/drm/asahi/m3/m3_completion.rs | 66 + drivers/gpu/drm/asahi/m3/m3_compute.rs | 379 ++++ drivers/gpu/drm/asahi/m3/m3_config.rs | 444 +++++ drivers/gpu/drm/asahi/m3/m3_device.rs | 315 ++++ drivers/gpu/drm/asahi/m3/m3_drm.rs | 210 +++ drivers/gpu/drm/asahi/m3/m3_firmware.rs | 441 +++++ drivers/gpu/drm/asahi/m3/m3_memory.rs | 163 ++ drivers/gpu/drm/asahi/m3/m3_params.rs | 900 ++++++++++ drivers/gpu/drm/asahi/m3/m3_pass.rs | 51 + drivers/gpu/drm/asahi/m3/m3_pool.rs | 41 + drivers/gpu/drm/asahi/m3/m3_render.rs | 696 +++++++ drivers/gpu/drm/asahi/m3/m3_resources.rs | 51 + drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 327 ++++ drivers/gpu/drm/asahi/m3/m3_runtime.rs | 1246 +++++++++++++ drivers/gpu/drm/asahi/m3/m3_submit.rs | 297 +++ drivers/gpu/drm/asahi/m3/m3_thermal.rs | 499 +++++ drivers/gpu/drm/asahi/m3/t8122_knobs.rs | 333 ++++ drivers/gpu/drm/asahi/m3/t8122_start.rs | 579 ++++++ 34 files changed, 14629 insertions(+) create mode 100644 drivers/gpu/drm/asahi/m3/agx_attachments.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_compute.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_fault.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_memory.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_resources.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_status.rs create mode 100644 drivers/gpu/drm/asahi/m3/agx_uapi.rs create mode 100644 drivers/gpu/drm/asahi/m3/g15_raw.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw/agx3.rs create mode 100644 drivers/gpu/drm/asahi/m3/hw/t8122.rs create mode 100644 drivers/gpu/drm/asahi/m3/identity.rs create mode 100644 drivers/gpu/drm/asahi/m3/initdata.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_adt_config.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_board.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_client.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_completion.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_compute.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_config.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_device.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_drm.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_firmware.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_memory.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_params.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_pass.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_pool.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_render.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_resources.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_rtkit.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_runtime.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_submit.rs create mode 100644 drivers/gpu/drm/asahi/m3/m3_thermal.rs create mode 100644 drivers/gpu/drm/asahi/m3/t8122_knobs.rs create mode 100644 drivers/gpu/drm/asahi/m3/t8122_start.rs diff --git a/drivers/gpu/drm/asahi/m3/agx_attachments.rs b/drivers/gpu/drm/asahi/m3/agx_attachments.rs new file mode 100644 index 00000000000000..5aeb3c7335c222 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_attachments.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +pub(crate) const SIZE: usize = 0x104; +pub(crate) const CAPACITY: usize = 16; + +#[derive(Clone, Copy)] +pub(crate) struct Attachments(pub(crate) [u8; SIZE]); + +impl Attachments { + pub(crate) const EMPTY: Self = Self([0; SIZE]); + pub(crate) fn count(&self) -> u8 { self.0[0x100] } + + /// Normalize byte ranges to the firmware's 128-byte cache-line units. + /// The caller must validate the entire normalized writable GPU range. + pub(crate) fn range(address: u64, size: u64) -> Result<(u64, u64), ()> { + if address == 0 || size == 0 { return Err(()); } + let end = address.checked_add(size).and_then(|v| v.checked_add(127)).ok_or(())? & !127; + let start = address & !127; + if start == 0 || end > 1 << 42 || (end - start) >> 7 > u32::MAX as u64 { + return Err(()); + } + Ok((start, end - start)) + } + + pub(crate) fn new(ranges: &[(u64, u64)]) -> Result { + if ranges.len() > CAPACITY { return Err(()); } + let mut out = Self::EMPTY; + for (record, &(address, size)) in out.0[..0x100].chunks_exact_mut(16).zip(ranges) { + let (address, size) = Self::range(address, size)?; + record[..8].copy_from_slice(&address.to_le_bytes()); + record[8..12].copy_from_slice(&((size >> 7) as u32).to_le_bytes()); + record[12..14].copy_from_slice(&0x18u16.to_le_bytes()); + record[14..16].copy_from_slice(&1u16.to_le_bytes()); + } + out.0[0x100..].copy_from_slice(&(ranges.len() as u32).to_le_bytes()); + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn covers_both_partial_edge_lines() { + let a = Attachments::new(&[(0x100007f, 2)]).unwrap(); + assert_eq!(u64::from_le_bytes(a.0[..8].try_into().unwrap()), 0x1000000); + assert_eq!(u32::from_le_bytes(a.0[8..12].try_into().unwrap()), 2); + assert_eq!(&a.0[12..16], &[0x18, 0, 1, 0]); + assert_eq!(a.count(), 1); + assert!(a.0[16..0x100].iter().all(|v| *v == 0)); + } + #[test] + fn rejects_unrepresentable_ranges_and_count() { + for range in [(0, 128), (128, 0), (u64::MAX - 1, 8), + ((1 << 42) - 128, 129), (128, (u32::MAX as u64 + 1) << 7)] { + assert!(Attachments::new(&[range]).is_err()); + } + assert!(Attachments::new(&[(128, 128); CAPACITY + 1]).is_err()); + let full = Attachments::new(&[(128, 128); CAPACITY]).unwrap(); + assert_eq!(full.count(), 16); + assert_eq!(&full.0[0x100..], &[16, 0, 0, 0]); + assert_eq!(Attachments::new(&[]).unwrap().0, Attachments::EMPTY.0); + } + #[test] + fn matches_native_m4_writable_buffer_descriptor() { + let encoded = Attachments::new(&[(0x100000a8000, 0x400000)]).unwrap(); + assert_eq!(&encoded.0[..16], &[ + 0, 0x80, 0x0a, 0, 0, 1, 0, 0, 0, 0x80, 0, 0, 0x18, 0, 1, 0]); + assert_eq!(encoded.count(), 1); + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_compute.rs b/drivers/gpu/drm/asahi/m3/agx_compute.rs new file mode 100644 index 00000000000000..d58005c504f7ac --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_compute.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +/// Complete client CDM range and its USC address window. +#[derive(Clone, Copy)] +pub(crate) struct Control { + pub base: u64, + pub end: u64, + pub usc_base: u64, +} diff --git a/drivers/gpu/drm/asahi/m3/agx_fault.rs b/drivers/gpu/drm/asahi/m3/agx_fault.rs new file mode 100644 index 00000000000000..9c37cfa98dce9f --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_fault.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Immutable fault-only snapshots of owned M3 firmware memory. +//! +//! Records contain name[32], firmware VA, length, begin/end monotonic ns, +//! then exactly `length` bytes. No MMIO or application BOs are read. Reads +//! are sequential observations, not atomic with running firmware. + +use kernel::{prelude::*, time::{ClockSource, Monotonic}}; + +pub(crate) struct Dump { + bytes: KVVec, + count: u32, + limit: usize, +} + +impl Dump { + + /// M3 snapshots have no scheduler job ID yet and contain only bounded + /// driver-owned firmware configuration and channel storage. + pub(crate) fn new_m3() -> Result { + Self::with_format(0, b"M3FWD001", 2 * 1024 * 1024) + } + + fn with_format(job: u64, magic: &[u8; 8], limit: usize) -> Result { + // Reserve before firmware/job publication, while healthy. Record appends + // must never enter reclaim waiting on the GPU that just faulted. + let mut bytes = KVVec::with_capacity(limit, GFP_KERNEL)?; + bytes.resize(40, 0, GFP_KERNEL)?; + bytes[..8].copy_from_slice(magic); + bytes[16..24].copy_from_slice(&job.to_le_bytes()); + bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); + Ok(Self { bytes, count: 0, limit }) + } + + /// Start the one-shot capture; construction may precede the fault by hours. + pub(crate) fn begin(&mut self, job: u64) { + self.bytes[16..24].copy_from_slice(&job.to_le_bytes()); + self.bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); + } + + pub(crate) fn record(&mut self, name: &str, va: u64, size: usize, + read: impl FnOnce(&mut [u8]) -> Result) -> Result { + if name.is_empty() || name.len() >= 32 || size == 0 { return Err(EINVAL); } + let count = self.count.checked_add(1).ok_or(EOVERFLOW)?; + let start = self.bytes.len(); + let end = start.checked_add(64).and_then(|x| x.checked_add(size)).ok_or(EOVERFLOW)?; + if end > self.limit { return Err(E2BIG); } + // Capacity was reserved at construction. NOWAIT is defense in depth; + // the checked limit above guarantees resize cannot allocate here. + self.bytes.resize(end, 0, GFP_NOWAIT)?; + self.bytes[start..start+name.len()].copy_from_slice(name.as_bytes()); + self.bytes[start+32..start+40].copy_from_slice(&va.to_le_bytes()); + self.bytes[start+40..start+48].copy_from_slice(&(size as u64).to_le_bytes()); + let begin = Monotonic::ktime_get() as u64; + if let Err(error) = read(&mut self.bytes[start+64..end]) { + self.bytes.truncate(start); + return Err(error); + } + let finish = Monotonic::ktime_get() as u64; + self.bytes[start+48..start+56].copy_from_slice(&begin.to_le_bytes()); + self.bytes[start+56..start+64].copy_from_slice(&finish.to_le_bytes()); + self.count = count; + Ok(()) + } + + pub(crate) fn publish(mut self, dev: &kernel::device::Device) -> Result { + let length = self.bytes.len(); + self.bytes[8..12].copy_from_slice(&self.count.to_le_bytes()); + self.bytes[12..16].copy_from_slice(&1u32.to_le_bytes()); + self.bytes[32..40].copy_from_slice(&(length as u64).to_le_bytes()); + let format = "M3FWD001"; + let owned = KBox::new(self, GFP_NOWAIT)?; + kernel::devcoredump::dev_coredump(dev, &crate::THIS_MODULE, owned, GFP_NOWAIT, + kernel::devcoredump::DEFAULT_TIMEOUT); + dev_info!(dev, "Asahi: sealed {}-byte firmware fault snapshot offered to devcoredump ({})\n", length, format); + Ok(()) + } +} + +impl kernel::devcoredump::DevCoreDump for Dump { + fn read(&self, output: &mut [u8], offset: usize) -> Result { + if offset >= self.bytes.len() { return Ok(0); } + let length = output.len().min(self.bytes.len()-offset); + output[..length].copy_from_slice(&self.bytes[offset..offset+length]); + Ok(length) + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_memory.rs b/drivers/gpu/drm/asahi/m3/agx_memory.rs new file mode 100644 index 00000000000000..521990b4aad214 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_memory.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +// Firmware validates Timestamp user destinations against a dedicated 64 MiB +// interval published in HwDataB+0x28. Reserve the last 64 MiB of the +// firmware VM selector, disjoint from MMIO/command allocations above. +pub(crate) const TIMESTAMP_RANGE: core::ops::Range = + 0xffff_fc2f_fc00_0000..0xffff_fc30_0000_0000; + +pub(crate) fn publish() { + // SAFETY: Orders preceding WC buffer stores before a device doorbell. + unsafe { core::arch::asm!("dsb oshst", options(nostack, preserves_flags)) }; +} diff --git a/drivers/gpu/drm/asahi/m3/agx_resources.rs b/drivers/gpu/drm/asahi/m3/agx_resources.rs new file mode 100644 index 00000000000000..8fdf87bdb046dd --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_resources.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! J713 bootloader resource handoff. Reading this metadata does not map memory, +//! claim UAT ownership, authenticate firmware, or grant permission to start it. +//! Firmware ABI admission and an owned runtime are separate steps. + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Region { + pub(crate) base: u64, + pub(crate) size: u64, +} + +impl Region { + fn end(self) -> Option { + self.base.checked_add(self.size) + } + + fn valid(self) -> bool { + self.base != 0 + && self.size != 0 + && (self.base | self.size) & 0x3fff == 0 + && self.end().is_some_and(|end| end <= 1 << 42) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ResourceError { + Version, + Region, + Overlap, + FirmwareLayout, +} + +#[derive(Debug)] +pub(crate) struct Resources { + /// ttbs, pagetables, handoff, shared-l2, fw-text, fw-data. + pub(crate) regions: [Region; 6], + pub(crate) firmware_vas: [u64; 2], +} + +impl Resources { + pub(crate) fn validate( + version: u32, + regions: [Region; 6], + firmware_vas: [u64; 2], + firmware_flags: [u32; 2], + ) -> Result { + if version != 1 { + return Err(ResourceError::Version); + } + for (i, region) in regions.iter().enumerate() { + if !region.valid() { + return Err(ResourceError::Region); + } + for other in ®ions[..i] { + if region.base < other.end().unwrap() && other.base < region.end().unwrap() { + return Err(ResourceError::Overlap); + } + } + } + // The inherited 42-bit page-table roots and all 64 16-byte TTBAT + // entries fit their reserved regions. No panel-sized allocation pool. + if firmware_flags != [1, 0] + || firmware_vas[0] != 0xffff_fc00_0000_0000 + || firmware_vas[0].checked_add(regions[4].size) != Some(firmware_vas[1]) + || firmware_vas[1].checked_add(regions[5].size).is_none() + { + return Err(ResourceError::FirmwareLayout); + } + Ok(Self { + regions, + firmware_vas, + }) + } +} + + + +#[cfg(test)] +mod tests { + use super::*; + + fn observed() -> [Region; 6] { + [ + (0x103fffb8000, 0x4000), + (0x103fff78000, 0x40000), + (0x103fff70000, 0x4000), + (0x103fff74000, 0x4000), + (0x10000cc0000, 0x60000), + (0x10001df4000, 0x13c000), + ] + .map(|(base, size)| Region { base, size }) + } + const VAS: [u64; 2] = [0xfffffc0000000000, 0xfffffc0000060000]; + + #[test] + fn observed_and_relocated_boot_addresses() { + let res = Resources::validate(1, observed(), VAS, [1, 0]).unwrap(); + assert_eq!(res.firmware_vas, VAS); + assert_eq!(res.regions[2].base, 0x103fff70000); + let relocated = observed().map(|r| Region { + base: r.base + 0x400000000, + ..r + }); + assert!(Resources::validate(1, relocated, VAS, [1, 0]).is_ok()); + } + + /// Relocating the UAT reservations preserves their sizes and ownership checks. + #[test] + fn relocated_uat_regions_keep_firmware_segments() { + let mut regions = observed(); + for region in &mut regions[..4] { + region.base -= 0x40000; + } + let res = Resources::validate(1, regions, VAS, [1, 0]).unwrap(); + assert_eq!(res.regions[0].base, 0x103fff78000); + assert_eq!(res.regions[4], observed()[4]); + assert_eq!(res.regions[5], observed()[5]); + } + + #[test] + fn rejects_missing_overlapping_unaligned_and_out_of_range_memory() { + for r in [ + Region { base: 0, size: 0 }, + Region { + base: 0x12345, + size: 0x4000, + }, + Region { + base: 1 << 42, + size: 0x4000, + }, + Region { + base: u64::MAX & !0x3fff, + size: 0x4000, + }, + ] { + let mut bad = observed(); + bad[0] = r; + assert!(matches!( + Resources::validate(1, bad, VAS, [1, 0]), + Err(ResourceError::Region) + )); + } + let mut bad = observed(); + bad[0] = bad[1]; + assert!(matches!( + Resources::validate(1, bad, VAS, [1, 0]), + Err(ResourceError::Overlap) + )); + } + + #[test] + fn rejects_unknown_handoff_and_segment_layouts() { + assert!(matches!( + Resources::validate(0, observed(), VAS, [1, 0]), + Err(ResourceError::Version) + )); + for (vas, flags) in [ + (VAS, [0, 1]), + ([VAS[0], VAS[1] + 0x4000], [1, 0]), + ([0xfffffe0000000000, VAS[1]], [1, 0]), + ] { + assert!(matches!( + Resources::validate(1, observed(), vas, flags), + Err(ResourceError::FirmwareLayout) + )); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/agx_status.rs b/drivers/gpu/drm/asahi/m3/agx_status.rs new file mode 100644 index 00000000000000..1d83086a1d155f --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_status.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! CPU-owned sticky VM failure reporting. No GPU mapping or firmware access. +use core::sync::atomic::{AtomicI32, Ordering}; + +pub(crate) const PARAM_GROUP_VM_STATUS: u32 = 1; +pub(crate) const FEATURE_VM_STATUS: u64 = 1 << 1; +pub(crate) const VM_STATUS_SIZE: usize = 16; + +/// Shared by the VM and accepted jobs; never cleared by queue destruction or +/// successful work. A newly created VM receives a new status object. +pub(crate) struct VmStatus { error: AtomicI32 } +impl VmStatus { + pub(crate) fn new() -> Self { Self { error: AtomicI32::new(0) } } + pub(crate) fn record(&self, error: i32) { + if error < 0 { + let _ = self.error.compare_exchange(0, error, Ordering::AcqRel, Ordering::Acquire); + } + } + pub(crate) fn get(&self) -> i32 { self.error.load(Ordering::Acquire) } +} + +/// GET_PARAMS group 1 is an in/out request: vm_id, flags=0, error(out), pad=0. +/// Decode once, before lookup, so a racing userspace edit cannot change owner. +pub(crate) fn decode_request(bytes: &[u8; VM_STATUS_SIZE]) -> Option { + let word = |i| u32::from_ne_bytes(bytes[i..i+4].try_into().unwrap()); + if word(4) != 0 || word(12) != 0 { return None; } + Some(word(0)) +} +pub(crate) fn encode_response(vm_id: u32, error: i32) -> [u8; VM_STATUS_SIZE] { + let mut bytes = [0; VM_STATUS_SIZE]; + bytes[0..4].copy_from_slice(&vm_id.to_ne_bytes()); + bytes[8..12].copy_from_slice(&error.to_ne_bytes()); + bytes +} diff --git a/drivers/gpu/drm/asahi/m3/agx_uapi.rs b/drivers/gpu/drm/asahi/m3/agx_uapi.rs new file mode 100644 index 00000000000000..60107f8833edc2 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/agx_uapi.rs @@ -0,0 +1,721 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +#![cfg_attr(not(test), allow(dead_code))] + +//! Shared modern-Asahi UAPI parsing. +//! +//! The byte layouts match `drm_asahi_cmd_header`, `drm_asahi_cmd_render`, +//! `drm_asahi_cmd_compute`, and `drm_asahi_attachment`. Translation validates +//! the queue's 4 GiB USC window and every userspace range before constructing +//! M3 render parameters or compute-entry operands. This module allocates no +//! object, publishes no queue state, and performs no firmware or MMIO action. + +pub(crate) const UAPI_COMMAND_HEADER_SIZE: usize = 8; +pub(crate) const UAPI_RENDER_SIZE: usize = 256; +pub(crate) const UAPI_COMPUTE_SIZE: usize = 64; +pub(crate) const UAPI_ATTACHMENT_SIZE: usize = 24; +pub(crate) const UAPI_MAX_ATTACHMENTS: usize = 16; +pub(crate) const UAPI_MAX_HARDWARE_COMMANDS: u32 = 64; +pub(crate) const UAPI_BARRIER_NONE: u16 = 0xffff; +pub(crate) const USC_WINDOW_SIZE: u64 = 1 << 32; +const COMMAND_RENDER: u16 = 0; +const COMMAND_COMPUTE: u16 = 1; +const SET_VERTEX_ATTACHMENTS: u16 = 2; +const SET_FRAGMENT_ATTACHMENTS: u16 = 3; +const SET_COMPUTE_ATTACHMENTS: u16 = 4; + +const RENDER_VERTEX_SCRATCH: u32 = 1 << 0; +const RENDER_PROCESS_EMPTY_TILES: u32 = 1 << 1; +const RENDER_NO_VERTEX_CLUSTERING: u32 = 1 << 2; +const RENDER_RSRC_SPEC_HI: u32 = 1 << 4; +const RENDER_DBIAS_IS_INT: u32 = 1 << 18; +const RENDER_FLAGS: u32 = RENDER_VERTEX_SCRATCH + | RENDER_PROCESS_EMPTY_TILES + | RENDER_NO_VERTEX_CLUSTERING + | RENDER_RSRC_SPEC_HI + | RENDER_DBIAS_IS_INT; + +const SAMPLER_SIZE: u64 = 8; +const MAX_SAMPLERS: u32 = 1024; +const PAGE_SIZE: u64 = 0x4000; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum UapiTranslateError { + HeaderTruncated, + PayloadTruncated, + PayloadTail, + CommandType, + AttachmentPayloadSize, + AttachmentCount, + AttachmentFields, + AttachmentBarrier, + HardwareCommandCount, + BarrierFuture, + QueueUscAlignment, + QueueUscRange, + AddressRange, + AddressAlignment, + FlagsUnknownBits, + FlagsUnsupported, + Dimensions, + Layers, + UtileDimensions, + Samples, + TilebufferSize, + SamplerCount, + SamplerPair, + HelperProgram, + Add3ProofAttachments, + ProgramOffset, + ZlsFields, + StencilClearBits, +} + +impl UapiTranslateError { + /// Stable short name for one rejection, so a failing submit reports which + /// check refused it instead of a bare EINVAL. Kept as `&'static str` (no + /// `Debug` formatting) to stay cheap on the ioctl path. + pub(crate) fn name(&self) -> &'static str { + match self { + Self::HeaderTruncated => "header-truncated", + Self::PayloadTruncated => "payload-truncated", + Self::PayloadTail => "payload-tail", + Self::CommandType => "command-type", + Self::AttachmentPayloadSize => "attachment-payload-size", + Self::AttachmentCount => "attachment-count", + Self::AttachmentFields => "attachment-fields", + Self::AttachmentBarrier => "attachment-barrier", + Self::HardwareCommandCount => "hardware-command-count", + Self::BarrierFuture => "barrier-future", + Self::QueueUscAlignment => "queue-usc-alignment", + Self::QueueUscRange => "queue-usc-range", + Self::AddressRange => "address-range", + Self::AddressAlignment => "address-alignment", + Self::FlagsUnknownBits => "flags-unknown-bits", + Self::FlagsUnsupported => "flags-unsupported", + Self::Dimensions => "dimensions", + Self::Layers => "layers", + Self::UtileDimensions => "utile-dimensions", + Self::Samples => "samples", + Self::TilebufferSize => "tilebuffer-size", + Self::SamplerCount => "sampler-count", + Self::SamplerPair => "sampler-pair", + Self::HelperProgram => "helper-program", + Self::Add3ProofAttachments => "add3-proof-attachments", + Self::ProgramOffset => "program-offset", + Self::ZlsFields => "zls-fields", + Self::StencilClearBits => "stencil-clear-bits", + + + } + } +} + +fn get_u16(raw: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(raw[offset..offset + 2].try_into().unwrap()) +} + +fn get_u32(raw: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(raw[offset..offset + 4].try_into().unwrap()) +} + +fn get_u64(raw: &[u8], offset: usize) -> u64 { + u64::from_le_bytes(raw[offset..offset + 8].try_into().unwrap()) +} + +fn padded_payload(raw: &[u8]) -> Result<[u8; N], UapiTranslateError> { + if raw.len() > N && raw[N..].iter().any(|byte| *byte != 0) { + return Err(UapiTranslateError::PayloadTail); + } + let mut out = [0u8; N]; + let count = core::cmp::min(raw.len(), N); + out[..count].copy_from_slice(&raw[..count]); + Ok(out) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiCommandHeader { + pub(crate) command_type: u16, + pub(crate) size: u16, + pub(crate) render_barrier: u16, + pub(crate) compute_barrier: u16, +} + +impl UapiCommandHeader { + fn parse(raw: &[u8]) -> Result { + if raw.len() < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + Ok(Self { + command_type: get_u16(raw, 0), + size: get_u16(raw, 2), + render_barrier: get_u16(raw, 4), + compute_barrier: get_u16(raw, 6), + }) + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +pub(crate) struct UapiAttachment { + pub(crate) address: u64, + pub(crate) size: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiAttachmentList { + pub(crate) entries: [UapiAttachment; UAPI_MAX_ATTACHMENTS], + pub(crate) count: u8, +} + +impl UapiAttachmentList { + pub(crate) const EMPTY: Self = Self { + entries: [UapiAttachment { + address: 0, + size: 0, + }; UAPI_MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[UapiAttachment] { + &self.entries[..self.count as usize] + } + + fn parse(raw: &[u8]) -> Result { + if raw.len() % UAPI_ATTACHMENT_SIZE != 0 { + return Err(UapiTranslateError::AttachmentPayloadSize); + } + let count = raw.len() / UAPI_ATTACHMENT_SIZE; + if count > UAPI_MAX_ATTACHMENTS { + return Err(UapiTranslateError::AttachmentCount); + } + let mut out = Self::EMPTY; + for index in 0..count { + let offset = index * UAPI_ATTACHMENT_SIZE; + if get_u32(raw, offset + 0x10) != 0 || get_u32(raw, offset + 0x14) != 0 { + return Err(UapiTranslateError::AttachmentFields); + } + out.entries[index] = UapiAttachment { + address: get_u64(raw, offset), + size: get_u64(raw, offset + 8), + }; + } + out.count = count as u8; + Ok(out) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiHelperProgram { + pub(crate) binary: u32, + pub(crate) config: u32, + pub(crate) data: u64, +} + +fn helper(raw: &[u8], offset: usize) -> UapiHelperProgram { + UapiHelperProgram { + binary: get_u32(raw, offset), + config: get_u32(raw, offset + 4), + data: get_u64(raw, offset + 8), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiProgram { + pub(crate) usc: u32, + pub(crate) resource_spec: u64, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamp { + pub(crate) handle: u32, + pub(crate) offset: u32, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiTimestamps { + pub(crate) start: UapiTimestamp, + pub(crate) end: UapiTimestamp, +} + +fn timestamps(raw: &[u8], offset: usize) -> UapiTimestamps { + UapiTimestamps { + start: UapiTimestamp { + handle: get_u32(raw, offset), + offset: get_u32(raw, offset + 4), + }, + end: UapiTimestamp { + handle: get_u32(raw, offset + 8), + offset: get_u32(raw, offset + 12), + }, + } +} + +fn program(raw: &[u8], offset: usize, high_offset: usize) -> UapiProgram { + UapiProgram { + usc: get_u32(raw, offset), + resource_spec: get_u32(raw, offset + 4) as u64 + | (get_u32(raw, high_offset) as u64) << 32, + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiZlsBuffer { + pub(crate) base: u64, + pub(crate) compression_base: u64, + pub(crate) stride: u32, + pub(crate) compression_stride: u32, +} + +fn zls(raw: &[u8], offset: usize) -> UapiZlsBuffer { + UapiZlsBuffer { + base: get_u64(raw, offset), + compression_base: get_u64(raw, offset + 8), + stride: get_u32(raw, offset + 0x10), + compression_stride: get_u32(raw, offset + 0x14), + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiRenderCommand { + pub(crate) flags: u32, + pub(crate) depth_dimensions: u32, + pub(crate) vdm_base: u64, + pub(crate) vertex_helper: UapiHelperProgram, + pub(crate) fragment_helper: UapiHelperProgram, + pub(crate) scissor_base: u64, + pub(crate) depth_bias_base: u64, + pub(crate) occlusion_query_base: u64, + pub(crate) depth: UapiZlsBuffer, + pub(crate) stencil: UapiZlsBuffer, + pub(crate) zls_control: u64, + pub(crate) multisample_control: u64, + pub(crate) sampler_heap: u64, + pub(crate) ppp_control: u32, + pub(crate) width: u16, + pub(crate) height: u16, + pub(crate) layers: u16, + pub(crate) sampler_count: u16, + pub(crate) utile_width: u8, + pub(crate) utile_height: u8, + pub(crate) samples: u8, + pub(crate) sample_size: u8, + pub(crate) merge_upper_x: u32, + pub(crate) merge_upper_y: u32, + pub(crate) background: UapiProgram, + pub(crate) end_of_tile: UapiProgram, + pub(crate) partial_background: UapiProgram, + pub(crate) partial_end_of_tile: UapiProgram, + pub(crate) depth_clear: u32, + pub(crate) stencil_clear: u32, + pub(crate) vertex_timestamps: UapiTimestamps, + pub(crate) fragment_timestamps: UapiTimestamps, +} + +impl UapiRenderCommand { + fn parse(raw: &[u8]) -> Result { + let raw = padded_payload::(raw)?; + Ok(Self { + flags: get_u32(&raw, 0x00), + depth_dimensions: get_u32(&raw, 0x04), + vdm_base: get_u64(&raw, 0x08), + vertex_helper: helper(&raw, 0x10), + fragment_helper: helper(&raw, 0x20), + scissor_base: get_u64(&raw, 0x30), + depth_bias_base: get_u64(&raw, 0x38), + occlusion_query_base: get_u64(&raw, 0x40), + depth: zls(&raw, 0x48), + stencil: zls(&raw, 0x60), + zls_control: get_u64(&raw, 0x78), + multisample_control: get_u64(&raw, 0x80), + sampler_heap: get_u64(&raw, 0x88), + ppp_control: get_u32(&raw, 0x90), + width: get_u16(&raw, 0x94), + height: get_u16(&raw, 0x96), + layers: get_u16(&raw, 0x98), + sampler_count: get_u16(&raw, 0x9a), + utile_width: raw[0x9c], + utile_height: raw[0x9d], + samples: raw[0x9e], + sample_size: raw[0x9f], + merge_upper_x: get_u32(&raw, 0xa0), + merge_upper_y: get_u32(&raw, 0xa4), + background: program(&raw, 0xa8, 0xf8), + end_of_tile: program(&raw, 0xb0, 0xf0), + partial_background: program(&raw, 0xb8, 0xfc), + partial_end_of_tile: program(&raw, 0xc0, 0xf4), + depth_clear: get_u32(&raw, 0xc8), + stencil_clear: get_u32(&raw, 0xcc), + vertex_timestamps: timestamps(&raw, 0xd0), + fragment_timestamps: timestamps(&raw, 0xe0), + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct UapiComputeCommand { + pub(crate) flags: u32, + pub(crate) sampler_count: u32, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) sampler_heap: u64, + pub(crate) helper: UapiHelperProgram, + pub(crate) timestamps: UapiTimestamps, +} + +impl UapiComputeCommand { + fn parse(raw: &[u8]) -> Result { + let raw = padded_payload::(raw)?; + Ok(Self { + flags: get_u32(&raw, 0), + sampler_count: get_u32(&raw, 4), + control_stream_base: get_u64(&raw, 8), + control_stream_end: get_u64(&raw, 0x10), + sampler_heap: get_u64(&raw, 0x18), + helper: helper(&raw, 0x20), + timestamps: timestamps(&raw, 0x30), + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum ParsedHardwareCommand { + Render { + index: u32, + header: UapiCommandHeader, + payload: UapiRenderCommand, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + }, + Compute { + index: u32, + header: UapiCommandHeader, + payload: UapiComputeCommand, + attachments: UapiAttachmentList, + }, +} + +pub(crate) struct UapiCommandParser<'a> { + raw: &'a [u8], + offset: usize, + command_index: u32, + hardware_count: u32, + render_count: u16, + compute_count: u16, + vertex_attachments: UapiAttachmentList, + fragment_attachments: UapiAttachmentList, + compute_attachments: UapiAttachmentList, +} + +impl<'a> UapiCommandParser<'a> { + pub(crate) const fn new(raw: &'a [u8]) -> Self { + Self { + raw, + offset: 0, + command_index: 0, + hardware_count: 0, + render_count: 0, + compute_count: 0, + vertex_attachments: UapiAttachmentList::EMPTY, + fragment_attachments: UapiAttachmentList::EMPTY, + compute_attachments: UapiAttachmentList::EMPTY, + } + } + + pub(crate) fn finish(&self) -> Result { + if self.offset != self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + if self.hardware_count == 0 || self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + Ok(self.hardware_count) + } + + pub(crate) fn next_hardware( + &mut self, + ) -> Result, UapiTranslateError> { + loop { + if self.offset == self.raw.len() { + return Ok(None); + } + if self.raw.len() - self.offset < UAPI_COMMAND_HEADER_SIZE { + return Err(UapiTranslateError::HeaderTruncated); + } + let header = UapiCommandHeader::parse(&self.raw[self.offset..])?; + self.offset += UAPI_COMMAND_HEADER_SIZE; + let end = self + .offset + .checked_add(header.size as usize) + .ok_or(UapiTranslateError::PayloadTruncated)?; + if end > self.raw.len() { + return Err(UapiTranslateError::PayloadTruncated); + } + let payload = &self.raw[self.offset..end]; + self.offset = end; + self.command_index += 1; + + match header.command_type { + COMMAND_RENDER | COMMAND_COMPUTE => { + if header.render_barrier != UAPI_BARRIER_NONE + && header.render_barrier > self.render_count + { + return Err(UapiTranslateError::BarrierFuture); + } + if header.compute_barrier != UAPI_BARRIER_NONE + && header.compute_barrier > self.compute_count + { + return Err(UapiTranslateError::BarrierFuture); + } + self.hardware_count += 1; + if self.hardware_count > UAPI_MAX_HARDWARE_COMMANDS { + return Err(UapiTranslateError::HardwareCommandCount); + } + if header.command_type == COMMAND_RENDER { + self.render_count += 1; + return Ok(Some(ParsedHardwareCommand::Render { + index: self.command_index, + header, + payload: UapiRenderCommand::parse(payload)?, + vertex_attachments: self.vertex_attachments, + fragment_attachments: self.fragment_attachments, + })); + } + self.compute_count += 1; + return Ok(Some(ParsedHardwareCommand::Compute { + index: self.command_index, + header, + payload: UapiComputeCommand::parse(payload)?, + attachments: self.compute_attachments, + })); + } + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if header.render_barrier != UAPI_BARRIER_NONE + || header.compute_barrier != UAPI_BARRIER_NONE + { + return Err(UapiTranslateError::AttachmentBarrier); + } + let attachments = UapiAttachmentList::parse(payload)?; + match header.command_type { + SET_VERTEX_ATTACHMENTS => self.vertex_attachments = attachments, + SET_FRAGMENT_ATTACHMENTS => self.fragment_attachments = attachments, + SET_COMPUTE_ATTACHMENTS => self.compute_attachments = attachments, + _ => unreachable!(), + } + } + _ => return Err(UapiTranslateError::CommandType), + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) enum GpuAccess { + Read, + Write, + ReadWrite, +} + +pub(crate) trait GpuAddressSpace { + fn covers(&self, address: u64, size: u64, access: GpuAccess) -> bool; +} + +fn require_range( + address_space: &A, + address: u64, + size: u64, + access: GpuAccess, +) -> Result<(), UapiTranslateError> { + let end = address + .checked_add(size) + .ok_or(UapiTranslateError::AddressRange)?; + if size == 0 || end <= address || !address_space.covers(address, size, access) { + return Err(UapiTranslateError::AddressRange); + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct QueueUscWindow { + pub(crate) base: u64, + pub(crate) user_start: u64, + pub(crate) user_end: u64, +} + +impl QueueUscWindow { + pub(crate) fn validate(self) -> Result<(), UapiTranslateError> { + if self.base & (USC_WINDOW_SIZE - 1) != 0 { + return Err(UapiTranslateError::QueueUscAlignment); + } + let end = self + .base + .checked_add(USC_WINDOW_SIZE) + .ok_or(UapiTranslateError::QueueUscRange)?; + if self.base < self.user_start || end > self.user_end || end <= self.base { + return Err(UapiTranslateError::QueueUscRange); + } + Ok(()) + } + + pub(crate) fn full_program( + self, + address_space: &A, + tagged_offset: u32, + ) -> Result { + self.validate()?; + if tagged_offset == 0 { + return Err(UapiTranslateError::ProgramOffset); + } + let allocation = self + .base + .checked_add((tagged_offset & !7) as u64) + .ok_or(UapiTranslateError::ProgramOffset)?; + require_range(address_space, allocation, 4, GpuAccess::Read)?; + Ok(allocation) + } +} + +fn validate_sampler( + address_space: &A, + heap: u64, + count: u32, +) -> Result<(), UapiTranslateError> { + if count > MAX_SAMPLERS { + return Err(UapiTranslateError::SamplerCount); + } + if (heap == 0) != (count == 0) { + return Err(UapiTranslateError::SamplerPair); + } + if count != 0 { + if heap & (SAMPLER_SIZE - 1) != 0 { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + heap, + count as u64 * SAMPLER_SIZE, + GpuAccess::Read, + )?; + } + Ok(()) +} + +pub(crate) fn validate_attachments( + address_space: &A, + attachments: &UapiAttachmentList, +) -> Result<(), UapiTranslateError> { + for attachment in attachments.as_slice() { + require_range( + address_space, + attachment.address, + attachment.size, + GpuAccess::Write, + )?; + } + Ok(()) +} + +pub(crate) fn validate_zls( + address_space: &A, + value: UapiZlsBuffer, + layers: u16, +) -> Result<(), UapiTranslateError> { + if value.base == 0 + && (value.compression_base != 0 || value.stride != 0 || value.compression_stride != 0) + { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_base == 0 && value.compression_stride != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if layers > 1 && value.base != 0 && value.stride == 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.stride != 0 && value.stride & 0x3fff != 1 { + return Err(UapiTranslateError::ZlsFields); + } + if value.compression_stride & 0x3fff != 0 { + return Err(UapiTranslateError::ZlsFields); + } + if value.base != 0 { + let layer_stride = if value.stride == 0 { + 0 + } else { + ((value.stride as u64 >> 14) + 1) * PAGE_SIZE + }; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range(address_space, value.base, size, GpuAccess::ReadWrite)?; + } + if value.compression_base != 0 { + let layer_stride = ((value.compression_stride as u64 >> 14) + 1) * 0x80; + let size = layer_stride + .checked_mul(layers.saturating_sub(1) as u64) + .and_then(|value| value.checked_add(1)) + .ok_or(UapiTranslateError::AddressRange)?; + require_range( + address_space, + value.compression_base, + size, + GpuAccess::ReadWrite, + )?; + } + Ok(()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub(crate) struct TranslatedComputeCommand { + pub(crate) usc_exec_base: u64, + pub(crate) control_stream_base: u64, + pub(crate) control_stream_end: u64, + pub(crate) control_stream_terminator: u64, + pub(crate) sampler_heap: u64, + pub(crate) sampler_count: u32, + pub(crate) attachments: UapiAttachmentList, + pub(crate) timestamps: UapiTimestamps, +} + +pub(crate) fn translate_compute_command( + command: UapiComputeCommand, + attachments: UapiAttachmentList, + queue: QueueUscWindow, + address_space: &A, +) -> Result { + queue.validate()?; + if command.flags != 0 { + return Err(UapiTranslateError::FlagsUnknownBits); + } + if command.control_stream_base & 3 != 0 + || command.control_stream_end & 3 != 0 + || command.control_stream_end <= command.control_stream_base + { + return Err(UapiTranslateError::AddressAlignment); + } + require_range( + address_space, + command.control_stream_base, + command.control_stream_end - command.control_stream_base, + GpuAccess::Read, + )?; + validate_sampler(address_space, command.sampler_heap, command.sampler_count)?; + if command.helper + != (UapiHelperProgram { + binary: 0, + config: 0, + data: 0, + }) + { + return Err(UapiTranslateError::HelperProgram); + } + Ok(TranslatedComputeCommand { + usc_exec_base: queue.base, + control_stream_base: command.control_stream_base, + control_stream_end: command.control_stream_end, + control_stream_terminator: command.control_stream_end - 4, + sampler_heap: command.sampler_heap, + sampler_count: command.sampler_count, + attachments, + timestamps: command.timestamps, + }) +} diff --git a/drivers/gpu/drm/asahi/m3/g15_raw.rs b/drivers/gpu/drm/asahi/m3/g15_raw.rs new file mode 100644 index 00000000000000..a876cad1ec283c --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/g15_raw.rs @@ -0,0 +1,1800 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU initialization / global structures + +use super::channels; +use super::types::*; +use crate::{ + default_zeroed, + gem, + mmu, + no_debug, + trivial_gpustruct, // +}; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct ChannelRing { + pub(crate) state: Option>, + pub(crate) ring: Option>, + } + + /// Host->FW ring descriptor of the 14.x firmware ABI. + /// + /// Instead of one `ChannelState` pointer, each host->FW ring (pipes, DeviceControl) is + /// described by the FW VAs of its three index words and of the ring itself. On G15 the + /// indices (and the pipe rings) live in Fender scratch SRAM. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct ScratchRingDesc { + /// FW VA of the u32 read index (FW writes; scratch part B). +0x00 + pub(crate) read_index: U64, + /// FW VA of the u32 "CFI" index (FW writes; scratch part B). +0x08 + pub(crate) cfi_index: U64, + /// FW VA of the u32 write index (host writes; scratch part A). +0x10 + pub(crate) write_index: U64, + /// FW VA of the ring entries. +0x18 + pub(crate) ring: U64, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PipeChannelsG15V14_8_3 { + + // 14.x ABI: {TA, 3D, CL} descriptors of 0x20 bytes, 0x60 per priority + pub(crate) vtx: ScratchRingDesc, + pub(crate) frag: ScratchRingDesc, + pub(crate) comp: ScratchRingDesc, + } + default_zeroed!(PipeChannelsG15V14_8_3); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatusFlags { + pub(crate) halt_count: AtomicU64, + __pad0: Pad<0x8>, + pub(crate) halted: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) resume: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) unk_40: u32, + __pad3: Pad<0xc>, + pub(crate) unk_ctr: u32, + /// P0+0x45c4. Non-zero makes the firmware install its work-scheduler callbacks + /// Left 0, the first compute kick branches to address 0 (observed on J516S). + pub(crate) sched_callbacks: u32, + __pad4: Pad<0x8>, + pub(crate) unk_60: u32, + __pad5: Pad<0xc>, + pub(crate) unk_70: u32, + __pad6: Pad<0xc>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatus { + pub(crate) fwctl_channel: ChannelRing, + pub(crate) flags: FwStatusFlags, + } + + /// G15 "P0" globals-2/status block, 0xc3d0 bytes, pointed to by InitData+0xb0. It embeds the + /// G13 `FwStatus` at +0x4568 (FWCtl ring pointers, 8 bytes + /// of padding, then the halt/resume flags at +0x4580), which is why the + /// FWCtl ring and flags keep their `FwStatus` field names here. + #[repr(C)] + pub(crate) struct StatusBlock { + /// 0 at init; also written by the firmware register dump. +0x0 + pub(crate) unk_0: u32, + /// Noise-suppression idle power-off state. +0x4 + pub(crate) noise_suppression_idle_pwroff: u32, + /// 0 at init. +0x8 + pub(crate) unk_8: u32, + /// Not touched by the host (0x0008..0x4017). + /// TODO: firmware-private, or tables (pending stamps?) the host must fill? + pub(crate) unk_c: Array<0x400c, u8>, + /// Perf-counter sampler control, 0x4018..0x40a0. Unused by drm/asahi. + pub(crate) perfctr: Array<0x88, u8>, + /// FW-initiated recovery info (FW-written). +0x40a0/+0x40a4 + pub(crate) fw_recovery_info_a: u32, + pub(crate) fw_recovery_info_b: u32, + pub(crate) unk_40a8: Array<0x3f8, u8>, + /// Recovery-packet details read on event type 4. +0x44a0 + pub(crate) recovery_packet_info: Array<0x14, u8>, + pub(crate) unk_44b4: Array<0xc, u8>, + /// Read while draining events. +0x44c0 + pub(crate) unk_44c0: U64, + pub(crate) unk_44c8: Array<0x90, u8>, + /// Read while draining events. +0x4558/+0x4560 + pub(crate) unk_4558: U64, + pub(crate) unk_4560: U64, + /// FWCtl `{state, ring}` FW VAs. +0x4568/+0x4570 + pub(crate) fwctl_channel: ChannelRing, + /// 8 bytes of padding put the flags on a 16-byte boundary. + pub(crate) __pad_4578: Pad<0x8>, + /// halt_count/halted/resume/... at +0x4580/+0x4590/+0x45a0. + /// +0x45c4 (`flags.sched_callbacks`) must be non-zero or the work scheduler + /// is left without its callbacks. + pub(crate) flags: FwStatusFlags, + /// Not touched by the host (0x45c8..0xc3c7). + pub(crate) unk_45f0: Array<0x7dd8, u8>, + /// QoS mode, 0 at init. +0xc3c8 + pub(crate) qos_mode: u32, + /// TODO: read-modify-written at shared-data init; left 0. +0xc3cc + pub(crate) unk_c3cc: u32, + } + default_zeroed!(StatusBlock); + no_debug!(StatusBlock); + + /// G15 "P1" power-controller shared block, 0x238 bytes, InitData+0xb8. Its 0x0c..0xd4 part + /// keeps the order of the G13 13.5 `Globals` power + /// fields (power zones, fast-die and PPM controllers), so the same field names are used. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PowerCtlBlock { + /// FW-published power values. +0x0/+0x4 + pub(crate) fw_power_a: u32, + pub(crate) fw_power_b: u32, + /// Power-zone reset request. +0x8 + pub(crate) power_zone_reset: u32, + /// +0xc..+0x5f: G13 Globals `power_zone_count`..`power_zones`, same order. + pub(crate) power_zone_count: u32, + pub(crate) avg_power_filter_tc_periods: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) avg_power_target_filter_tc: u32, + pub(crate) power_zones: Array<5, PowerZoneGlobal>, + /// CLVR loop override / control effort. +0x60..+0x68 + pub(crate) clvr_loop_override_a: u32, + pub(crate) clvr_ctrl_effort: u32, + pub(crate) clvr_loop_override_b: u32, + pub(crate) unk_6c: Array<0x38, u8>, + /// Lifetime-servo temperature target. +0xa4 + pub(crate) lifetime_servo_temp_a: u32, + /// +0xa8..+0xc8: G13 Globals `unk_89bc`..`unk_89e0`, same order. + pub(crate) fast_die_temp_target: u32, + pub(crate) fast_die0_release_temp: u32, + pub(crate) unk_b0: i32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_kp: F32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: u32, + /// +0xc8..+0xdb: the fast-die/PPM enable flags, the PPM target (maximum power) and the + /// PPM controller gains, in G13 Globals order (`unk_89e0`, `max_power_2`, `ppm_kp`, + /// `ppm_ki_dt`). The two T6030 backends place them differently: the manager starts + /// with the enable flag at +0xc8 (the layout it was validated with), the runtime backend + /// with 0 at +0xc8 and the flag at +0xcc, as its InitData has them. See the builder. + pub(crate) ppm_words: Array<5, u32>, + pub(crate) unk_dc: Array<0xc, u8>, + /// "SE" controller override / engagement enable. +0xe8/+0xec + pub(crate) se_controller_override: u32, + pub(crate) se_engagement: u32, + pub(crate) unk_f0: Array<0x68, u8>, + /// DPE leakage update config. +0x158..+0x1ab + pub(crate) dpe_leak_cfg_a: u32, + pub(crate) unk_15c: u32, + pub(crate) unk_160: u32, + pub(crate) unk_164: u32, + pub(crate) dpe_leak_coefs: Array<0x40, u8>, + pub(crate) dpe_leak_cfg_b: u32, + pub(crate) unk_1ac: Array<0x8, u8>, + /// CLPC shared data (FW-written). +0x1b4..+0x1cc + pub(crate) clpc_shared_a: U64, + pub(crate) clpc_shared_b: U64, + pub(crate) clpc_split_ratio_a: u32, + pub(crate) clpc_split_ratio_b: u32, + pub(crate) clpc_shared_c: u32, + /// +0x1d0 (u8) / +0x1d1 (unaligned u32) + pub(crate) perf_ctrl_override_active: u8, + pub(crate) consistent_perf_state: U32, + pub(crate) unk_1d5: Array<0x3, u8>, + /// FW-written energy / filtered power. +0x1d8/+0x1e0/+0x1e4 + pub(crate) accumulated_energy: U64, + pub(crate) filtered_power: u32, + pub(crate) unk_1e4: u32, + /// SLC DSID configuration. +0x1e8..+0x22b + pub(crate) dsid_cfg_a: U64, + pub(crate) unk_1f0: Array<0x28, u8>, + pub(crate) dsid_cfg_b: U64, + pub(crate) dsid_cfg_c: U64, + pub(crate) dsid_cfg_d: u32, + pub(crate) unk_22c: u32, + /// +0x230 + pub(crate) submit_in_progress_gate: u32, + pub(crate) unk_234: u32, + } + default_zeroed!(PowerCtlBlock); + + /// G15 "P2" debug/trace block, 0x20 bytes, InitData+0xa8. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct DebugBlock { + /// Moved here from G13 `Globals.ktrace_enable`. +0x0 + pub(crate) ktrace_enable: u32, + /// TODO: written at init with an unknown value; left 0. +0x4 + pub(crate) unk_4: u32, + /// System-sleep notification in progress. +0x8 + pub(crate) system_sleep_in_progress: u32, + pub(crate) unk_c: u32, + /// Firmware-written. Read-only for the host. + pub(crate) fw_word_10: AtomicU32, + /// Firmware init progress: the firmware writes 0 once it consumes the INIT handshake and 1 + /// once its post-INIT initialisation is done, each followed by a barrier. The G15 driver + /// writes a sentinel here before MSG_INIT so the three states (not consumed / stuck / done) + /// are distinguishable. + pub(crate) init_state: AtomicU32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + } + default_zeroed!(DebugBlock); + + /// One Globals register-override record, 0x18 bytes. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct RegisterOverride { + pub(crate) addr: U64, + pub(crate) value: U64, + pub(crate) id: u32, + pub(crate) __pad: u32, + } + + /// One InitSeq register-initialisation record, 0x18 bytes. + /// + /// The InitSeq buffer (InitData+0x08) is a list of these. A zeroed buffer is a valid empty + /// list, which is what drm/asahi passes; this type only documents the format. + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + #[allow(dead_code)] + pub(crate) struct InitSeqRecord { + pub(crate) value: U64, + pub(crate) reg_id: u32, + pub(crate) extra: u32, + /// 1 = 32-bit, 2 = 64-bit, 3 = 64-bit physical address. + pub(crate) kind: u32, + pub(crate) __pad: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared1 { + pub(crate) table: Array<16, i32>, + pub(crate) unk_44: Array<0x60, u8>, + pub(crate) unk_a4: u32, + pub(crate) unk_a8: u32, + } + default_zeroed!(HwDataShared1); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2Curve { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) t1: Array<16, u16>, + pub(crate) t2: Array<16, i16>, + pub(crate) t3: Array<8, Array<16, i32>>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2G14 { + pub(crate) unk_0: Array<5, u32>, + pub(crate) unk_14: u32, + pub(crate) unk_18: Array<8, u32>, + pub(crate) curve1: HwDataShared2Curve, + pub(crate) curve2: HwDataShared2Curve, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared2 { + pub(crate) table: Array<10, i32>, + pub(crate) unk_28: Array<0x10, u8>, + pub(crate) g14: HwDataShared2G14, + pub(crate) unk_500: u32, + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + } + default_zeroed!(HwDataShared2); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared3 { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) table: Array<16, u32>, + pub(crate) unk_4c: u32, + } + default_zeroed!(HwDataShared3); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA130Extra { + pub(crate) unk_0: Array<0x38, u8>, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) unk_44: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_78: F32, + pub(crate) unk_7c: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_88: u32, + pub(crate) unk_8c: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) unk_94: u32, + pub(crate) unk_98: u32, + pub(crate) unk_9c: F32, + pub(crate) unk_a0: u32, + pub(crate) unk_a4: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: F32, + pub(crate) unk_c8: Array<0x4c, u8>, + pub(crate) unk_114: F32, + pub(crate) unk_118: u32, + pub(crate) unk_11c: u32, + pub(crate) unk_120: u32, + pub(crate) unk_124: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_12c: Array<0x8c, u8>, + } + default_zeroed!(HwDataA130Extra); + + /// G15 (14.x) shader-engine controller block of HwDataA, at +0x10b4. + /// + /// It holds the fields of [`HwDataA130Extra`] in a different order: the filter, gain and + /// time-constant words move 0x14 bytes down, and the threshold/criteria header moves behind + /// them, to +0xd8. Offsets in the field names are G15 block offsets. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA140Extra { + pub(crate) unk_0: Array<0x40, u8>, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) unk_48: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_64: F32, + pub(crate) unk_68: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_74: u32, + pub(crate) unk_78: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) unk_84: u32, + pub(crate) unk_88: F32, + /// Shader-engine target (ADT gpu-se-tgt). + pub(crate) se_target: u32, + pub(crate) unk_90: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_ac: u32, + pub(crate) unk_b0: F32, + pub(crate) unk_b4: Array<0x24, u8>, + pub(crate) unk_d8: u32, + pub(crate) unk_dc: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) unk_e8: u32, + pub(crate) unk_ec: u32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_f4: Array<0x28, u8>, + pub(crate) unk_11c: F32, + pub(crate) unk_120: Array<0xc, u8>, + pub(crate) unk_12c: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_134: Array<0x84, u8>, + } + default_zeroed!(HwDataA140Extra); + + #[repr(C)] + pub(crate) struct T81xxData { + pub(crate) unk_d8c: u32, + pub(crate) unk_d90: u32, + pub(crate) unk_d94: u32, + pub(crate) unk_d98: u32, + pub(crate) unk_d9c: F32, + pub(crate) unk_da0: u32, + pub(crate) unk_da4: F32, + pub(crate) unk_da8: u32, + pub(crate) unk_dac: F32, + pub(crate) unk_db0: u32, + pub(crate) unk_db4: u32, + pub(crate) unk_db8: F32, + pub(crate) unk_dbc: F32, + pub(crate) unk_dc0: u32, + pub(crate) unk_dc4: u32, + pub(crate) unk_dc8: u32, + pub(crate) max_pstate_scaled: u32, + } + default_zeroed!(T81xxData); + + #[derive(Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct PowerZoneG15V14_8_3 { + pub(crate) val: F32, + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc_x4: u32, + pub(crate) filter_tc_xperiod: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + pub(crate) filter_a_neg: F32, + pub(crate) filter_a: F32, + pub(crate) pad: u32, + } + + // G15 HwDataA uses the G13 single-die layout with 8x8 coefficient tables at 0x970-0xb6f + //, so the G14X 16-core layout is G14X-only. + const MAX_CORES_PER_CLUSTERG15V14_8_3: usize = { + { + 8 + } + }; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct AuxLeakCoef { + pub(crate) afr_1: Array<2, F32>, + pub(crate) cs_1: Array<2, F32>, + pub(crate) afr_2: Array<2, F32>, + pub(crate) cs_2: Array<2, F32>, + } + + #[repr(C)] + pub(crate) struct HwDataAG15V14_8_3 { + pub(crate) unk_0: u32, + pub(crate) clocks_per_period: u32, + + pub(crate) clocks_per_period_2: u32, + + pub(crate) unk_8: u32, + pub(crate) pwr_status: AtomicU32, + pub(crate) unk_10: F32, + pub(crate) unk_14: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) actual_pstate: u32, + pub(crate) tgt_pstate: u32, + pub(crate) unk_30: u32, + pub(crate) cur_pstate: u32, + pub(crate) unk_38: u32, + + pub(crate) unk_3c_0: u32, + + pub(crate) base_pstate_scaled: u32, + pub(crate) unk_40: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) unk_48: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) freq_mhz: F32, + pub(crate) unk_54: Array<0x20, u8>, + + pub(crate) unk_74_0: u32, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_b4: Array<0x100, u8>, + pub(crate) unk_1b4: u32, + pub(crate) temp_c: u32, + pub(crate) avg_power_mw: u32, + pub(crate) update_ts: U64, + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: Array<0x478, u8>, + pub(crate) pad_644: Pad<0x8>, + pub(crate) unk_64c: u32, + pub(crate) unk_650: u32, + pub(crate) pad_654: u32, + pub(crate) pwr_filter_a_neg: F32, + pub(crate) pad_65c: u32, + pub(crate) pwr_filter_a: F32, + pub(crate) pad_664: u32, + pub(crate) pwr_integral_gain: F32, + pub(crate) pad_66c: u32, + pub(crate) pwr_integral_min_clamp: F32, + pub(crate) max_power_1: F32, + pub(crate) pwr_proportional_gain: F32, + pub(crate) pad_67c: u32, + pub(crate) pwr_pstate_related_k: F32, + pub(crate) pwr_pstate_max_dc_offset: i32, + pub(crate) unk_688: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) pad_690: u32, + pub(crate) unk_694: u32, + pub(crate) max_power_2: u32, + pub(crate) pad_69c: Pad<0x18>, + pub(crate) unk_6b4: u32, + + pub(crate) unk_6b8_0: Array<0x10, u8>, + + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_6bc: u32, + pub(crate) pad_6c0: Pad<0x14>, + pub(crate) ppm_filter_tc_periods_x4: u32, + pub(crate) unk_6d8: u32, + pub(crate) pad_6dc: u32, + pub(crate) ppm_filter_a_neg: F32, + pub(crate) pad_6e4: u32, + pub(crate) ppm_filter_a: F32, + pub(crate) pad_6ec: u32, + pub(crate) ppm_ki_dt: F32, + pub(crate) pad_6f4: u32, + pub(crate) pwr_integral_min_clamp_2: u32, + pub(crate) unk_6fc: F32, + pub(crate) ppm_kp: F32, + pub(crate) pad_704: u32, + pub(crate) unk_708: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_4: u32, + pub(crate) unk_714: u32, + pub(crate) pad_718: u32, + pub(crate) unk_71c: F32, + pub(crate) max_power_3: u32, + pub(crate) cur_power_mw_2: u32, + pub(crate) ppm_filter_tc_ms: u32, + pub(crate) unk_72c: u32, + + pub(crate) ppm_filter_tc_clks: u32, + + pub(crate) unk_730_4: u32, + + pub(crate) unk_730_8: u32, + + pub(crate) unk_730_c: u32, + + pub(crate) unk_730: F32, + pub(crate) unk_734: u32, + pub(crate) unk_738: u32, + pub(crate) unk_73c: u32, + pub(crate) unk_740: u32, + pub(crate) unk_744: u32, + pub(crate) unk_748: Array<0x4, F32>, + pub(crate) unk_758: u32, + pub(crate) perf_tgt_utilization: u32, + pub(crate) pad_760: u32, + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_boost_ce_step: u32, + pub(crate) perf_reset_iters: u32, + pub(crate) pad_770: u32, + pub(crate) unk_774: u32, + pub(crate) unk_778: u32, + pub(crate) perf_filter_drop_threshold: u32, + pub(crate) perf_filter_a_neg: F32, + pub(crate) perf_filter_a2_neg: F32, + pub(crate) perf_filter_a: F32, + pub(crate) perf_filter_a2: F32, + pub(crate) perf_ki: F32, + pub(crate) perf_ki2: F32, + pub(crate) perf_integral_min_clamp: F32, + pub(crate) unk_79c: F32, + pub(crate) perf_kp: F32, + pub(crate) perf_kp2: F32, + pub(crate) boost_state_unk_k: F32, + pub(crate) base_pstate_scaled_2: u32, + pub(crate) max_pstate_scaled_5: u32, + pub(crate) base_pstate_scaled_3: u32, + pub(crate) pad_7b8: u32, + pub(crate) perf_cur_utilization: F32, + pub(crate) perf_tgt_utilization_2: u32, + pub(crate) pad_7c4: Pad<0x18>, + pub(crate) unk_7dc: u32, + + pub(crate) unk_7e0_0: Array<0x10, u8>, + + pub(crate) base_pstate_scaled_4: u32, + pub(crate) pad_7e4: u32, + pub(crate) unk_7e8: Array<0x14, u8>, + pub(crate) unk_7fc: F32, + pub(crate) pwr_min_duty_cycle_2: F32, + pub(crate) max_pstate_scaled_6: F32, + pub(crate) max_freq_mhz: u32, + pub(crate) pad_80c: u32, + pub(crate) unk_810: u32, + pub(crate) pad_814: u32, + pub(crate) pwr_min_duty_cycle_3: u32, + pub(crate) unk_81c: u32, + pub(crate) pad_820: u32, + pub(crate) min_pstate_scaled_4: F32, + pub(crate) max_pstate_scaled_7: u32, + pub(crate) unk_82c: u32, + pub(crate) unk_alpha_neg: F32, + pub(crate) unk_alpha: F32, + pub(crate) unk_838: u32, + pub(crate) unk_83c: u32, + pub(crate) pad_840: Pad<0x2c>, + pub(crate) unk_86c: u32, + pub(crate) fast_die0_sensor_mask: U64, + pub(crate) fast_die0_release_temp_cc: u32, + pub(crate) unk_87c: i32, + pub(crate) unk_880: u32, + pub(crate) unk_884: u32, + pub(crate) pad_888: u32, + pub(crate) unk_88c: u32, + pub(crate) pad_890: u32, + pub(crate) unk_894: F32, + pub(crate) pad_898: u32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) pad_8a0: u32, + pub(crate) unk_8a4: u32, + pub(crate) unk_8a8: F32, + pub(crate) fast_die0_kp: F32, + pub(crate) pad_8b0: u32, + pub(crate) unk_8b4: u32, + pub(crate) pwr_min_duty_cycle_4: u32, + pub(crate) max_pstate_scaled_8: u32, + pub(crate) max_pstate_scaled_9: u32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) unk_8c8: u32, + pub(crate) unk_8cc: u32, + pub(crate) pad_8d0: Pad<0x14>, + + pub(crate) unk_8e4_0: Array<0x10, u8>, + + pub(crate) unk_8e4: u32, + pub(crate) unk_8e8: u32, + pub(crate) max_pstate_scaled_10: u32, + pub(crate) unk_8f0: u32, + pub(crate) unk_8f4: u32, + pub(crate) pad_8f8: u32, + pub(crate) pad_8fc: u32, + pub(crate) unk_900: Array<0x24, u8>, + + pub(crate) unk_coef_a1: Array<8, Array>, + pub(crate) unk_coef_a2: Array<8, Array>, + + pub(crate) pad_b24: Pad<0x70>, + pub(crate) max_pstate_scaled_11: u32, + pub(crate) freq_with_off: u32, + pub(crate) unk_b9c: u32, + pub(crate) unk_ba0: U64, + pub(crate) unk_ba8: U64, + pub(crate) unk_bb0: u32, + pub(crate) unk_bb4: u32, + + pub(crate) pad_bb8_0: Pad<0x200>, + pub(crate) pad_bb8_200: Pad<0x8>, + + pub(crate) pad_bb8: Pad<0x74>, + pub(crate) unk_c2c: u32, + pub(crate) power_zone_count: u32, + pub(crate) max_power_4: u32, + pub(crate) max_power_5: u32, + pub(crate) max_power_6: u32, + pub(crate) unk_c40: u32, + pub(crate) unk_c44: F32, + pub(crate) avg_power_target_filter_a_neg: F32, + pub(crate) avg_power_target_filter_a: F32, + pub(crate) avg_power_target_filter_tc_x4: u32, + pub(crate) avg_power_target_filter_tc_xperiod: u32, + + pub(crate) avg_power_target_filter_tc_clks: u32, + + pub(crate) unk_c58_4: u32, + + pub(crate) power_zones: Array<5, PowerZoneG15V14_8_3>, + pub(crate) avg_power_filter_tc_periods_x4: u32, + pub(crate) unk_cfc: u32, + pub(crate) unk_d00: u32, + pub(crate) avg_power_filter_a_neg: F32, + pub(crate) unk_d08: u32, + pub(crate) avg_power_filter_a: F32, + pub(crate) unk_d10: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) unk_d18: u32, + pub(crate) unk_d1c: u32, + pub(crate) unk_d20: F32, + pub(crate) avg_power_kp: F32, + pub(crate) unk_d28: u32, + pub(crate) unk_d2c: u32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_12: u32, + pub(crate) max_pstate_scaled_13: u32, + pub(crate) unk_d3c: u32, + pub(crate) max_power_7: F32, + pub(crate) max_power_8: u32, + pub(crate) unk_d48: u32, + pub(crate) avg_power_filter_tc_ms: u32, + pub(crate) unk_d50: u32, + + pub(crate) avg_power_filter_tc_clks: u32, + + pub(crate) unk_d54_4: Array<0xc, u8>, + + pub(crate) unk_d54: Array<0x10, u8>, + pub(crate) max_pstate_scaled_14: u32, + pub(crate) unk_d68: Array<0x24, u8>, + + pub(crate) t81xx_data: T81xxData, + + pub(crate) unk_dd0: Array<0x40, u8>, + + pub(crate) unk_e10_pad: Array<0x10, u8>, + pub(crate) unk_e10_0: HwDataA140Extra, + + pub(crate) unk_e10: Array<0xc, u8>, + + // G15 offsets match G13 13.5 up to ~0x1224 and are +0x10 from 0x1288 + // (`fast_die0_sensor_mask_2`, G13 0x1278). TODO: exact insertion point between + // 0x1224 and 0x1288 is not known; placed as late as possible. + pub(crate) unk_1278_g15: Array<0x10, u8>, + + pub(crate) fast_die0_sensor_mask_2: U64, + + pub(crate) unk_e24: u32, + pub(crate) unk_e28: u32, + pub(crate) unk_e2c: Pad<0x1c>, + pub(crate) unk_coef_b1: Array<8, Array>, + pub(crate) unk_coef_b2: Array<8, Array>, + + pub(crate) pad_1048: Pad<0x5e4>, + + pub(crate) fast_die0_sensor_mask_alt: U64, + + pub(crate) unk_163c: u32, + + pub(crate) unk_1640: Array<0x2000, u8>, + + pub(crate) unk_3640: u32, + pub(crate) unk_3644: u32, + pub(crate) hws1: HwDataShared1, + + pub(crate) unk_hws2: Array<16, u16>, + + pub(crate) hws2: HwDataShared2, + pub(crate) unk_3c00: u32, + pub(crate) unk_3c04: u32, + pub(crate) hws3: HwDataShared3, + pub(crate) unk_3c58: Array<0x3c, u8>, + pub(crate) unk_3c94: u32, + pub(crate) unk_3c98: U64, + pub(crate) unk_3ca0: U64, + pub(crate) unk_3ca8: U64, + pub(crate) unk_3cb0: U64, + pub(crate) ts_last_idle: U64, + pub(crate) ts_last_poweron: U64, + pub(crate) ts_last_poweroff: U64, + pub(crate) unk_3cd0: U64, + pub(crate) unk_3cd8: U64, + + pub(crate) unk_3ce0_0: u32, + + pub(crate) unk_3ce0: u32, + pub(crate) unk_3ce4: u32, + pub(crate) unk_3ce8: u32, + pub(crate) unk_3cec: u32, + pub(crate) unk_3cf0: u32, + pub(crate) core_leak_coef: Array<8, F32>, + pub(crate) sram_leak_coef: Array<8, F32>, + + pub(crate) aux_leak_coef: AuxLeakCoef, + pub(crate) unk_3d34_0: Array<0x18, u8>, + + pub(crate) unk_3d34: Array<0x38, u8>, + + // G15 tail, 0x422c..0x4360. + pub(crate) unk_422c: Array<0x20, u8>, + + /// System counter value at init. +0x424c + /// The runtime backend fills it; the manager leaves it 0, as validated. + pub(crate) init_timestamp: U64, + pub(crate) unk_4254: Array<0x40, u8>, + + /// +0x4294/+0x4298 + pub(crate) unk_4294: u32, + pub(crate) unk_4298: u32, + pub(crate) unk_429c: Array<0x8, u8>, + + /// Three 8-entry per-cluster tables. +0x42a4 + /// TODO: source values unknown; left 0. + pub(crate) cluster_tables: Array<3, Array<8, u32>>, + + /// Eight words, source unknown. +0x4304..+0x4320 + pub(crate) unk_4304: Array<8, u32>, + pub(crate) unk_4324: Array<0x1c, u8>, + + /// Read-modify-written at init. +0x4340 + pub(crate) unk_4340: U64, + pub(crate) unk_4348: Array<0x10, u8>, + + /// GPU keep-alive mode (read by the host). +0x4358 + pub(crate) gpu_keepalive_mode: u32, + pub(crate) unk_435c: u32, + } + default_zeroed!(HwDataAG15V14_8_3); + no_debug!(HwDataAG15V14_8_3); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C)] + pub(crate) struct IOMapping { + pub(crate) phys_addr: U64, + pub(crate) virt_addr: U64, + pub(crate) total_size: u32, + pub(crate) element_size: u32, + pub(crate) readwrite: U64, + } + + const IO_MAPPING_COUNTG15V14_8_3: usize = { + // 31 IOMapping slots (0x3e0 / 0x20) + { + 0x1f + } + }; + + /// A G15 performance-state table: the highest entry, then per entry the frequency (MHz) + /// and the core and SRAM voltages (mV), eight voltage columns each. + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBPStateTable { + pub(crate) max_pstate: u32, + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBAuxPStates { + pub(crate) cs_max_pstate: u32, + pub(crate) cs_frequencies: Array<0x10, u32>, + pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_unkpad: u32, + pub(crate) afr_max_pstate: u32, + pub(crate) afr_frequencies: Array<0x8, u32>, + pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_unkpad: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBG15V14_8_3 { + + pub(crate) unk_8: U64, + + pub(crate) unk_18: U64, + pub(crate) unk_20: U64, + pub(crate) unk_28: U64, + pub(crate) unk_30: U64, + pub(crate) timestamp_area_base: U64, + + /// G15: a pointer. +0x30 + /// TODO: what it points to; left 0. + pub(crate) unk_30_ptr: U64, + + // G15: two tables of 32 CSC matrices (3x4 i16) at 0x38..0x637 + pub(crate) yuv_matrices: Array<0x40, Array<3, Array<4, i16>>>, + + pub(crate) pad_1c8: Pad<0x8>, + pub(crate) io_mappings: Array, + + pub(crate) sgx_sram_ptr: U64, + + pub(crate) chip_id: u32, + pub(crate) unk_454: u32, + pub(crate) unk_458: u32, + pub(crate) unk_45c: u32, + pub(crate) unk_460: u32, + pub(crate) unk_464: u32, + pub(crate) unk_468: u32, + pub(crate) unk_46c: u32, + pub(crate) unk_470: u32, + pub(crate) unk_474: u32, + pub(crate) unk_478: u32, + pub(crate) unk_47c: u32, + pub(crate) unk_480: u32, + pub(crate) unk_484: u32, + pub(crate) unk_488: u32, + pub(crate) unk_48c: u32, + pub(crate) base_clock_khz: u32, + pub(crate) power_sample_period: u32, + + // G15 0xa70..0xb3f. The G13 13.5 0x9b0..0xa73 group, + // with 12 bytes inserted at an unknown position; the words are named by their G15 offset. + // TODO: map these to the G13 names (`num_frags`, `unk_524` secure cache flush, ...) + // once their values are known. + pub(crate) unk_a70: u32, + pub(crate) unk_a74: u32, + pub(crate) unk_a78: u32, + pub(crate) unk_a7c: U64, + pub(crate) unk_a84: u32, + pub(crate) unk_a88: u32, + pub(crate) unk_a8c: u32, + pub(crate) unk_a90: u32, + pub(crate) unk_a94: u32, + pub(crate) unk_a98: u32, + pub(crate) unk_a9c: u32, + pub(crate) unk_aa0: u32, + pub(crate) unk_aa4: u32, + pub(crate) unk_aa8: u32, + pub(crate) unk_aac: u32, + pub(crate) unk_ab0: u32, + pub(crate) unk_ab4: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: U64, + pub(crate) unk_ac4: u32, + pub(crate) unk_ac8: Array<0x4, u32>, + pub(crate) unk_ad8: u32, + pub(crate) unk_adc: u32, + pub(crate) unk_ae0: u32, + pub(crate) unk_ae4: u32, + pub(crate) unk_ae8: U64, + pub(crate) unk_af0: u32, + pub(crate) unk_af4: U64, + pub(crate) unk_afc: u32, + pub(crate) unk_b00: u32, + pub(crate) unk_b04: u32, + pub(crate) unk_b08: Array<0x4, u32>, + pub(crate) unk_b18: u32, + pub(crate) unk_b1c: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: U64, + pub(crate) unk_b2c: Array<0x14, u8>, + + pub(crate) unk_554: u32, + pub(crate) uat_ttb_base: U64, + pub(crate) gpu_core_id: u32, + pub(crate) gpu_rev_id: u32, + pub(crate) num_cores: u32, + pub(crate) max_pstate: u32, + + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + + + /// G15: second 16-entry frequency table in MHz (meaning inferred). +0xf9c + pub(crate) frequencies_2: Array<0x10, u32>, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_9f4: Array<0x10, u32>, + pub(crate) rel_max_powers: Array<0x10, u32>, + pub(crate) rel_boost_freqs: Array<0x10, u32>, + + pub(crate) unk_arr_0: Array<32, u32>, + + pub(crate) aux_ps: HwDataBAuxPStates, + + // G15 0x134c..0x17eb: a third p-state table at 0x134c, then the + // words before `timer_offset`. G15 is +4 relative to G13 here (+0xd0 at `timer_offset`). + + /// Third p-state table: the secondary (lowest frequency per voltage) table, with one + /// voltage column. +0x134c + pub(crate) aux_ps_3: HwDataBPStateTable, + pub(crate) aux_ps_3_pad: Array<0xc, u8>, + pub(crate) unk_179c: U64, + pub(crate) unk_17a4: Array<0x10, u8>, + pub(crate) unk_17b4: u32, + pub(crate) unk_17b8: u32, + pub(crate) unk_17bc: u32, + + /// Unit enable bit masks. +0x17c0/+0x17c8 + pub(crate) unit_mask_a: U64, + pub(crate) unit_mask_b: u32, + pub(crate) unk_17cc: u32, + pub(crate) unk_17d0: u32, + pub(crate) unk_17d4: Array<0x10, u8>, + pub(crate) unk_17e4: u32, + pub(crate) unk_17e8: u32, + + pub(crate) timer_offset: U64, + + // G15 0x17f4..0x1867. + pub(crate) unk_17f4: U64, + pub(crate) unk_17fc: u32, + pub(crate) unk_1800: u32, + pub(crate) unk_1804: u32, + pub(crate) unk_1808: u32, + pub(crate) unk_180c: u32, + + /// +0x1810 and +0x1814 are both 1 (written as one 64-bit store); the firmware copies them + /// to separate globals. + pub(crate) unk_1810: u32, + pub(crate) unk_1814: u32, + pub(crate) unk_1818: u32, + pub(crate) unk_181c: u32, + + /// 48 bytes of 0xff ("all units enabled" masks?). +0x1820 + pub(crate) all_ones_masks: Array<0x30, u8>, + + /// FSTP override enable. +0x1850 + pub(crate) fstp_override: u32, + pub(crate) unk_1854: U64, + pub(crate) unk_185c: u32, + pub(crate) unk_1860: u32, + pub(crate) unk_1864: u32, + } + default_zeroed!(HwDataBG15V14_8_3); + + #[derive(Debug)] + #[repr(C, packed)] + pub(crate) struct GpuStatsVtx { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsVtx); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsFragG15V14_8_3 { + // G15: the two -1 fields are at E2+0xc18/+0xc30 + pub(crate) unk1_0: Array<0xb10, u8>, + pub(crate) unk1: Array<0x100, u8>, + pub(crate) cur_stamp_id: i32, + pub(crate) unk2: Array<0x14, u8>, + pub(crate) unk_id: i32, + // G15 E2 is 0x1248 bytes in total + pub(crate) unk3: Array<0x614, u8>, + } + + impl Default for GpuStatsFragG15V14_8_3 { + fn default() -> Self { + Self { + unk1_0: Default::default(), + unk1: Default::default(), + cur_stamp_id: -1, + unk2: Default::default(), + unk_id: -1, + unk3: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsVtx { + pub(crate) total_cmds: u32, + pub(crate) stats: GpuStatsVtx, + } + default_zeroed!(GpuGlobalStatsVtx); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsFragG15V14_8_3 { + pub(crate) total_cmds: u32, + pub(crate) unk_4: u32, + pub(crate) stats: GpuStatsFragG15V14_8_3, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsComp { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsComp); + + // Not part of the 0x490-byte 14.x RuntimePointers, so the G15V14_8_3 + // instantiation is unused. + #[derive(Debug)] + #[repr(C)] + #[allow(dead_code)] + pub(crate) struct RuntimeScratchG15V14_8_3 { + pub(crate) unk_280: Array<0x6800, u8>, + pub(crate) unk_6a80: u32, + pub(crate) gpu_idle: u32, + pub(crate) unkpad_6a88: Pad<0x14>, + pub(crate) unk_6a9c: u32, + pub(crate) unk_ctr0: u32, + pub(crate) unk_ctr1: u32, + pub(crate) unk_6aa8: u32, + pub(crate) unk_6aac: u32, + pub(crate) unk_ctr2: u32, + pub(crate) unk_6ab4: u32, + pub(crate) unk_6ab8: u32, + pub(crate) unk_6abc: u32, + pub(crate) unk_6ac0: u32, + pub(crate) unk_6ac4: u32, + pub(crate) unk_ctr3: u32, + pub(crate) unk_6acc: u32, + pub(crate) unk_6ad0: u32, + pub(crate) unk_6ad4: u32, + pub(crate) unk_6ad8: u32, + pub(crate) unk_6adc: u32, + pub(crate) unk_6ae0: u32, + pub(crate) unk_6ae4: u32, + pub(crate) unk_6ae8: u32, + pub(crate) unk_6aec: u32, + pub(crate) unk_6af0: u32, + pub(crate) unk_ctr4: u32, + pub(crate) unk_ctr5: u32, + pub(crate) unk_6afc: u32, + pub(crate) pad_6b00: Pad<0x38>, + + pub(crate) pad_6b00_extra: Array<0x4800, u8>, + + pub(crate) unk_6b38: u32, + pub(crate) pad_6b3c: Pad<0x84>, + } + default_zeroed!(RuntimeScratchG15V14_8_3); + + #[repr(C)] + pub(crate) struct RuntimePointersG15V14_8_3<'a> { + // G15 / 14.8.3: 0x490 bytes, packed (HwDataA pointer at +0x441). GpuPointer/U64 are + // packed(1), so repr(C) gives the firmware offsets. + + /// +0x000 + pub(crate) hwdata_b: GpuPointer<'a, super::HwDataBG15V14_8_3>, + + /// Hardware-erratum (BRN) table. +0x008 + pub(crate) brn_table: GpuPointer<'a, &'a [u8]>, + + /// E7 (0x88 bytes, zeroed). +0x010 + pub(crate) unkptr_e7: GpuPointer<'a, &'a [u8]>, + + pub(crate) pipes: Array<4, PipeChannelsG15V14_8_3>, + + /// +0x198 + pub(crate) device_control: ScratchRingDesc, + pub(crate) event: ChannelRing, + pub(crate) fw_log: ChannelRing, + pub(crate) ktrace: ChannelRing, + pub(crate) stats: ChannelRing, + + + /// FWLog payload buffer (0x600 x 0xd8). +0x1f8 + pub(crate) fwlog_buf: Option>, + pub(crate) __pad_200: Pad<0x30>, + + /// 0 at init. +0x230 + pub(crate) unk_230: u32, + pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>, + pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFragG15V14_8_3>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, + + + /// E5 (0x60 bytes, zeroed; FW status word at +0). +0x24c + pub(crate) unkptr_e5: GpuPointer<'a, &'a [u8]>, + pub(crate) __pad_254: Pad<0x54>, + + /// Optional pointer, left 0. +0x2a8 + pub(crate) unkptr_2a8: U64, + + /// PB descriptor table GPU/FW VA. +0x2b0/+0x2b8 + pub(crate) buffer_mgr_ctl_gpu_addr: U64, + pub(crate) buffer_mgr_ctl_fw_addr: U64, + + /// UMA page-pool descriptor table GPU/FW VA. +0x2c0/+0x2c8 + pub(crate) uma_pool_desc_gpu_addr: U64, + pub(crate) uma_pool_desc_fw_addr: U64, + + /// Configuration words, left 0. +0x2d0/+0x2d4 + pub(crate) unk_2d0: u32, + pub(crate) unk_2d4: u32, + pub(crate) __pad_2d8: Pad<0xd8>, + + /// 0xff at init (context-switch histogram control). +0x3b0 + pub(crate) cswitch_hist_ctl: u8, + pub(crate) unk_3b1: Array<0x90, u8>, + + // G15: +0x441 (unaligned) + pub(crate) hwdata_a: GpuPointer<'a, super::HwDataAG15V14_8_3>, + + + /// 16 configuration bytes, left 0. +0x449 + pub(crate) unk_449: Array<0x10, u8>, + pub(crate) __pad_459: Pad<0x14>, + + /// Zero at init. +0x46d + pub(crate) unk_46d: Array<0x20, u8>, + pub(crate) __pad_48d: Pad<0x3>, + } + no_debug!(RuntimePointersG15V14_8_3<'_>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PendingStamp { + pub(crate) info: AtomicU32, + pub(crate) wait_value: AtomicU32, + } + default_zeroed!(PendingStamp); + + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct FaultInfo { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) queue_uuid: u32, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + } + default_zeroed!(FaultInfo); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct PowerZoneGlobal { + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc: u32, + } + default_zeroed!(PowerZoneGlobal); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GlobalsG15V14_8_3 { + // G15 / 14.8.3 layout: 0xe00 bytes, cached, packed. + // It re-packs the G13 field groups; the large G13 arrays are gone. + + /// Timer context-switch mode, 0 or 7. +0x0 + pub(crate) timer_cswitch: u32, + pub(crate) unk_4: u32, + + /// +0x8 + pub(crate) dm_pause_mode: u32, + pub(crate) unk_c: u32, + + /// +0x10 + pub(crate) dm_pause_timer: u32, + pub(crate) unk_14: u32, + + /// +0x18 + pub(crate) disable_cswitch: u32, + + /// Configuration word. +0x1c + pub(crate) unk_1c: u32, + + /// +0x20/+0x24 + pub(crate) relaxed_cl_kill_timeout: u32, + pub(crate) frg_task_timeout: u32, + + /// G13 `unk_24`. +0x28 + pub(crate) unk_28: u32, + + /// Debug flags bit 0 (G13 `debug`). +0x2c + pub(crate) debug: u32, + + /// +0x30 + pub(crate) smart_idle_off_enable: u32, + pub(crate) unk_34: u32, + + /// Constant 0x78. +0x38 + pub(crate) unk_38: u32, + + /// +0x3c/+0x40 + pub(crate) cpms_window_size: u32, + pub(crate) cpms_tfca_size: u32, + pub(crate) unk_44: u32, + + /// Kick channel QoS pair. +0x48/+0x4c + pub(crate) kick_channel_qos_b: u32, + pub(crate) kick_channel_qos_a: u32, + + /// G13 `unk_54`/`unk_56`/`unk_58` moved -0x14. +0x50..+0x54 (inferred) + pub(crate) unk_50: u16, + pub(crate) unk_52: u16, + pub(crate) unk_54: u16, + + /// G13 `unk_5a`/`unk_5e`/`unk_62` moved -0x14 (unaligned). +0x56..+0x5e + pub(crate) unk_56: U32, + pub(crate) unk_5a: U32, + pub(crate) cswitch_timer_multiplier: U32, + + /// +0x62 (G13 `unk_66`, tentative) + pub(crate) cdm_cs_mode_change: U32, + pub(crate) unk_66: Array<0xa, u8>, + + /// Command submission enable; must be non-zero to submit work. +0x70 + pub(crate) command_submission_enabled: u32, + + /// Pending-submission counter (G13 `pending_submissions`). +0x74 + pub(crate) pending_submissions: AtomicU32, + + /// +0x78..+0x80 + pub(crate) power_if_target: Array<3, u32>, + + /// +0x84..+0x8c + pub(crate) controller_override_0: u32, + pub(crate) controller_override_1: u32, + pub(crate) perf_state_cap: u32, + + /// Perf-controller block. +0x90..+0xc7 + pub(crate) perf_controller: Array<0x38, u8>, + + /// +0xc8/+0xcc + pub(crate) deadline_control_effort: u32, + pub(crate) deadline_controller_override: u32, + pub(crate) unk_d0: Array<0xa0, u8>, + + /// DPE (dynamic power estimation) block. +0x170..+0x69f + pub(crate) dpe: Array<0x530, u8>, + + /// +0x6a0 + pub(crate) dpe_sl_config: U64, + pub(crate) unk_6a8: Array<0x100, u8>, + + /// Smart idle-off config. +0x7a8 + pub(crate) smart_idle_off_cfg: Array<10, u32>, + + /// +0x7d0..+0x7dc + pub(crate) ut_engagement: u32, + pub(crate) clvr_engagement: u32, + pub(crate) keepalive_perf_threshold_rd: u32, + pub(crate) keepalive_off_threshold_rd: u32, + pub(crate) unk_7e0: u32, + + /// Register overrides. +0x7e4 + pub(crate) reg_overrides: Array<16, RegisterOverride>, + + /// +0x964 + pub(crate) reg_override_count: u32, + pub(crate) unk_968: u32, + + /// Display power-management params. +0x96c/+0x974 + pub(crate) dpm_param_a: U64, + pub(crate) dpm_param_b: U64, + + /// +0x97c..+0x984 + pub(crate) progress_check_interval_3d: u32, + pub(crate) progress_check_interval_ta: u32, + pub(crate) progress_check_interval_cl: u32, + + /// +0x988..+0x99c: the G13 `unk_1102c_0`..`unk_1102c` group (inferred) + pub(crate) unk_1102c_0: u32, + pub(crate) unk_1102c_4: u32, + pub(crate) unk_1102c_8: u32, + pub(crate) unk_1102c_c: u32, + pub(crate) unk_1102c_10: u32, + pub(crate) unk_1102c: u32, + + /// +0x9a0..+0x9a8 + pub(crate) idle_off_delay_ms: AtomicU32, + pub(crate) fender_idle_off_delay_ms: u32, + pub(crate) fw_early_wake_timeout_ms: u32, + + /// +0x9ac + pub(crate) gvdm_timer_interval: u32, + + /// +0x9b0/+0x9b4 + pub(crate) cl_context_switch_timeout_ms: u32, + pub(crate) cl_kill_timeout_ms: u32, + pub(crate) unk_9b8: u32, + + /// +0x9bc + pub(crate) cdm_backoff_timeout: u8, + pub(crate) unk_9bd: Array<0x4, u8>, + + /// +0x9c1/+0x9c3 + pub(crate) fwutil_default_fab_pstate: Array<0x2, u8>, + pub(crate) fwutil_timer_period: u8, + pub(crate) unk_9c4: Array<0x19, u8>, + + /// Unaligned. +0x9dd + pub(crate) mtr_sensor_ptd_override_mask: U64, + pub(crate) unk_9e5: Array<0x400, u8>, + + /// Read by the host when it creates command queues. +0xde5 + pub(crate) unk_de5: U32, + + /// Keep-alive overrides/thresholds. +0xde9..+0xdf5 + pub(crate) gpu_keepalive_override: U32, + pub(crate) gfxc_keepalive_override: U32, + pub(crate) keepalive_perf_threshold: U32, + pub(crate) keepalive_off_threshold: U32, + + /// Soft-fault settings (possibly G13 `fault_control`). +0xdf9 + pub(crate) soft_fault_settings: U32, + pub(crate) unk_dfd: Array<0x3, u8>, + + /// The G13 pending-stamp table is not in the 0xe00-byte G15 Globals. This + /// zero-length field keeps `GpuManager::mark_pending_events()` compiling as a no-op. + /// TODO: find the G15 pending-stamp table (possibly in P0 0x0008..0x4017). + pub(crate) pending_stamps: Array<0, PendingStamp>, + } + default_zeroed!(GlobalsG15V14_8_3); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct UatLevelInfo { + pub(crate) unk_3: u8, + pub(crate) unk_1: u8, + pub(crate) unk_2: u8, + pub(crate) index_shift: u8, + pub(crate) num_entries: u16, + pub(crate) unk_4: u16, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) index_mask: U64, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitDataG15V14_8_3<'a> { + pub(crate) ver_info: Array<0x4, u16>, + + pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointersG15V14_8_3>, + pub(crate) globals: GpuPointer<'a, super::GlobalsG15V14_8_3>, + + /// G15: no FwStatus pointer; u32 0 then u32 1 (possibly a host-mapped FW allocations + /// flag). +0x28/+0x2c + pub(crate) unk_28: u32, + pub(crate) unk_2c: u32, + // 0x30..0x97 on G15 is a 0x68-byte copy of the UAT pmap descriptor (+0x18..+0x7f). Its + // first 0x64 bytes have the G13 layout (3 levels, 0x20-byte records). + pub(crate) uat_page_size: u16, + pub(crate) uat_page_bits: u8, + pub(crate) uat_num_levels: u8, + pub(crate) uat_level_info: Array<0x3, UatLevelInfo>, + + + /// G15: last 4 bytes of the pmap descriptor copy (descriptor +0x7c). + /// TODO: meaning unknown; left 0. + pub(crate) unk_94: u32, + pub(crate) __pad_98: Pad<0x10>, + + /// P2 debug block (0x20). +0xa8 + pub(crate) debug_block: GpuPointer<'a, super::DebugBlock>, + + /// P0 status block (0xc3d0). +0xb0 + pub(crate) status_block: GpuPointer<'a, super::StatusBlock>, + + /// P1 power-controller block (0x238). +0xb8 + pub(crate) power_ctl_block: GpuPointer<'a, super::PowerCtlBlock>, + } +} + +#[derive(Debug)] +pub(crate) struct ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, +} + +impl ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + raw::ChannelRing { + state: Some(self.state.weak_pointer()), + ring: Some(self.ring.weak_pointer()), + } + } +} + +trivial_gpustruct!(FwStatus); +trivial_gpustruct!(StatusBlock); +trivial_gpustruct!(PowerCtlBlock); +trivial_gpustruct!(DebugBlock); +trivial_gpustruct!(GpuGlobalStatsVtx); +trivial_gpustruct!(GpuGlobalStatsFragG15V14_8_3); +trivial_gpustruct!(GpuStatsComp); + +trivial_gpustruct!(HwDataAG15V14_8_3); + +trivial_gpustruct!(HwDataBG15V14_8_3); + +#[derive(Debug)] +pub(crate) struct StatsG15V14_8_3 { + pub(crate) vtx: GpuObject, + pub(crate) frag: GpuObject, + pub(crate) comp: GpuObject, +} + +#[derive(Debug)] +pub(crate) struct RuntimePointersG15V14_8_3 { + pub(crate) stats: StatsG15V14_8_3, + + pub(crate) hwdata_a: GpuObject, + pub(crate) hwdata_b: GpuObject, + + // G15 group-A regions referenced from RuntimePointers + pub(crate) brn_table: GpuArray, + pub(crate) unkptr_e7: GpuArray, + pub(crate) unkptr_e5: GpuArray, + pub(crate) uma_pool_desc: GpuArray, + + pub(crate) buffer_mgr_ctl: gem::ObjectRef, + pub(crate) buffer_mgr_ctl_low_mapping: Option, + pub(crate) buffer_mgr_ctl_high_mapping: Option, +} + +impl GpuStruct for RuntimePointersG15V14_8_3 { + type Raw<'a> = raw::RuntimePointersG15V14_8_3<'a>; +} + +trivial_gpustruct!(GlobalsG15V14_8_3); + +#[derive(Debug)] +pub(crate) struct InitDataG15V14_8_3 { + pub(crate) unk_buf: GpuArray, + pub(crate) runtime_pointers: GpuObject, + pub(crate) globals: GpuObject, + + /// G15: the P0 status block, which embeds the FwStatus FWCtl ring and halt flags. It keeps + /// the `fw_status` name so the FwStatus users see the same fields. + pub(crate) fw_status: GpuObject, + pub(crate) power_ctl_block: GpuObject, + pub(crate) debug_block: GpuObject, +} + +impl GpuStruct for InitDataG15V14_8_3 { + type Raw<'a> = raw::InitDataG15V14_8_3<'a>; +} + +// G15 / 14.8.3 firmware structure layout checks: the sizes and offsets the firmware expects. +mod g15_layout { + use super::channels; + use super::raw::*; + use core::mem::{offset_of, size_of}; + use kernel::static_assert; + + // InitData, 0xc0 bytes used + static_assert!(size_of::>() == 0xc0); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, unk_buf) == 0x08); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, runtime_pointers) == 0x18); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, globals) == 0x20); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, unk_2c) == 0x2c); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, uat_page_size) == 0x30); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, debug_block) == 0xa8); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, status_block) == 0xb0); + static_assert!(offset_of!(InitDataG15V14_8_3<'static>, power_ctl_block) == 0xb8); + + // host->FW ring descriptor + static_assert!(size_of::() == 0x20); + static_assert!(size_of::() == 0x60); + + // RuntimePointers: 0x490 bytes, packed + static_assert!(size_of::>() == 0x490); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, hwdata_b) == 0x000); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, brn_table) == 0x008); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_e7) == 0x010); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, pipes) == 0x018); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, device_control) == 0x198); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, event) == 0x1b8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, fw_log) == 0x1c8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, ktrace) == 0x1d8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats) == 0x1e8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, fwlog_buf) == 0x1f8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_230) == 0x230); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_vtx) == 0x234); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_frag) == 0x23c); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, stats_comp) == 0x244); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_e5) == 0x24c); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unkptr_2a8) == 0x2a8); + static_assert!( + offset_of!(RuntimePointersG15V14_8_3<'static>, buffer_mgr_ctl_gpu_addr) == 0x2b0 + ); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, buffer_mgr_ctl_fw_addr) == 0x2b8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, uma_pool_desc_gpu_addr) == 0x2c0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, uma_pool_desc_fw_addr) == 0x2c8); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_2d0) == 0x2d0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, cswitch_hist_ctl) == 0x3b0); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, hwdata_a) == 0x441); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_449) == 0x449); + static_assert!(offset_of!(RuntimePointersG15V14_8_3<'static>, unk_46d) == 0x46d); + + // Globals: 0xe00 bytes, packed + static_assert!(size_of::() == 0xe00); + static_assert!(size_of::() == 0x18); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_28) == 0x28); + static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3_globals_debug::OFFSET); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_38) == 0x38); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_50) == 0x50); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_56) == 0x56); + static_assert!(offset_of!(GlobalsG15V14_8_3, cswitch_timer_multiplier) == 0x5e); + static_assert!(offset_of!(GlobalsG15V14_8_3, command_submission_enabled) == 0x70); + static_assert!(offset_of!(GlobalsG15V14_8_3, pending_submissions) == 0x74); + static_assert!(offset_of!(GlobalsG15V14_8_3, perf_controller) == 0x90); + static_assert!(offset_of!(GlobalsG15V14_8_3, deadline_control_effort) == 0xc8); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpe) == 0x170); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpe_sl_config) == 0x6a0); + static_assert!(offset_of!(GlobalsG15V14_8_3, smart_idle_off_cfg) == 0x7a8); + static_assert!(offset_of!(GlobalsG15V14_8_3, ut_engagement) == 0x7d0); + static_assert!(offset_of!(GlobalsG15V14_8_3, reg_overrides) == 0x7e4); + static_assert!(offset_of!(GlobalsG15V14_8_3, reg_override_count) == 0x964); + static_assert!(offset_of!(GlobalsG15V14_8_3, dpm_param_a) == 0x96c); + static_assert!(offset_of!(GlobalsG15V14_8_3, progress_check_interval_3d) == 0x97c); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_1102c_0) == 0x988); + static_assert!(offset_of!(GlobalsG15V14_8_3, idle_off_delay_ms) == 0x9a0); + static_assert!(offset_of!(GlobalsG15V14_8_3, fender_idle_off_delay_ms) == 0x9a4); + static_assert!(offset_of!(GlobalsG15V14_8_3, fw_early_wake_timeout_ms) == 0x9a8); + static_assert!(offset_of!(GlobalsG15V14_8_3, cl_context_switch_timeout_ms) == 0x9b0); + static_assert!(offset_of!(GlobalsG15V14_8_3, cdm_backoff_timeout) == 0x9bc); + static_assert!(offset_of!(GlobalsG15V14_8_3, fwutil_default_fab_pstate) == 0x9c1); + static_assert!(offset_of!(GlobalsG15V14_8_3, mtr_sensor_ptd_override_mask) == 0x9dd); + static_assert!(offset_of!(GlobalsG15V14_8_3, unk_de5) == 0xde5); + static_assert!(offset_of!(GlobalsG15V14_8_3, soft_fault_settings) == 0xdf9); + + // P0 status block + static_assert!(size_of::() == 0xc3d0); + static_assert!(offset_of!(StatusBlock, fw_recovery_info_a) == 0x40a0); + static_assert!(offset_of!(StatusBlock, recovery_packet_info) == 0x44a0); + static_assert!(offset_of!(StatusBlock, unk_44c0) == 0x44c0); + static_assert!(offset_of!(StatusBlock, unk_4558) == 0x4558); + static_assert!(offset_of!(StatusBlock, fwctl_channel) == 0x4568); + static_assert!(offset_of!(StatusBlock, flags) == 0x4580); + static_assert!(offset_of!(StatusBlock, flags.halted) == 0x4590); + static_assert!(offset_of!(StatusBlock, flags.resume) == 0x45a0); + static_assert!(offset_of!(StatusBlock, flags.unk_40) == 0x45b0); + static_assert!(offset_of!(StatusBlock, flags.sched_callbacks) == 0x45c4); + static_assert!(offset_of!(StatusBlock, qos_mode) == 0xc3c8); + + // P1 power-controller block + static_assert!(size_of::() == 0x238); + static_assert!(offset_of!(PowerCtlBlock, power_zone_count) == 0x0c); + static_assert!(offset_of!(PowerCtlBlock, power_zones) == 0x24); + static_assert!(offset_of!(PowerCtlBlock, clvr_loop_override_a) == 0x60); + static_assert!(offset_of!(PowerCtlBlock, lifetime_servo_temp_a) == 0xa4); + static_assert!(offset_of!(PowerCtlBlock, fast_die0_release_temp) == 0xac); + static_assert!(offset_of!(PowerCtlBlock, ppm_words) == 0xc8); + static_assert!(offset_of!(PowerCtlBlock, unk_dc) == 0xdc); + static_assert!(offset_of!(PowerCtlBlock, se_controller_override) == 0xe8); + static_assert!(offset_of!(PowerCtlBlock, se_engagement) == 0xec); + static_assert!(offset_of!(PowerCtlBlock, dpe_leak_cfg_a) == 0x158); + static_assert!(offset_of!(PowerCtlBlock, dpe_leak_cfg_b) == 0x1a8); + static_assert!(offset_of!(PowerCtlBlock, clpc_shared_a) == 0x1b4); + static_assert!(offset_of!(PowerCtlBlock, perf_ctrl_override_active) == 0x1d0); + static_assert!(offset_of!(PowerCtlBlock, consistent_perf_state) == 0x1d1); + static_assert!(offset_of!(PowerCtlBlock, accumulated_energy) == 0x1d8); + static_assert!(offset_of!(PowerCtlBlock, dsid_cfg_a) == 0x1e8); + static_assert!(offset_of!(PowerCtlBlock, dsid_cfg_b) == 0x218); + static_assert!(offset_of!(PowerCtlBlock, submit_in_progress_gate) == 0x230); + + // P2 debug block + static_assert!(size_of::() == 0x20); + static_assert!(offset_of!(DebugBlock, system_sleep_in_progress) == 0x08); + static_assert!(offset_of!(DebugBlock, fw_word_10) == 0x10); + static_assert!(offset_of!(DebugBlock, init_state) == 0x14); + + // InitSeq record + static_assert!(size_of::() == 0x18); + static_assert!(offset_of!(InitSeqRecord, kind) == 0x10); + + // HwDataB: 0x1868 bytes + static_assert!(size_of::() == 0x1868); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_30) == 0x20); + static_assert!(offset_of!(HwDataBG15V14_8_3, timestamp_area_base) == 0x28); + static_assert!(offset_of!(HwDataBG15V14_8_3, yuv_matrices) == 0x38); + static_assert!(offset_of!(HwDataBG15V14_8_3, pad_1c8) == 0x638); + static_assert!(offset_of!(HwDataBG15V14_8_3, io_mappings) == 0x640); + static_assert!(offset_of!(HwDataBG15V14_8_3, sgx_sram_ptr) == 0xa20); + static_assert!(offset_of!(HwDataBG15V14_8_3, chip_id) == 0xa28); + static_assert!(offset_of!(HwDataBG15V14_8_3, base_clock_khz) == 0xa68); + static_assert!(offset_of!(HwDataBG15V14_8_3, power_sample_period) == 0xa6c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_a9c) == 0xa9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_ac8) == 0xac8); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_b24) == 0xb24); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_554) == 0xb40); + static_assert!(offset_of!(HwDataBG15V14_8_3, uat_ttb_base) == 0xb44); + static_assert!(offset_of!(HwDataBG15V14_8_3, gpu_core_id) == 0xb4c); + static_assert!(offset_of!(HwDataBG15V14_8_3, num_cores) == 0xb54); + static_assert!(offset_of!(HwDataBG15V14_8_3, max_pstate) == 0xb58); + static_assert!(offset_of!(HwDataBG15V14_8_3, frequencies) == 0xb5c); + static_assert!(offset_of!(HwDataBG15V14_8_3, voltages) == 0xb9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, voltages_sram) == 0xd9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, frequencies_2) == 0xf9c); + static_assert!(offset_of!(HwDataBG15V14_8_3, sram_k) == 0xfdc); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_9f4) == 0x101c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_arr_0) == 0x10dc); + static_assert!(offset_of!(HwDataBG15V14_8_3, aux_ps) == 0x115c); + static_assert!(offset_of!(HwDataBG15V14_8_3, aux_ps_3) == 0x134c); + static_assert!(size_of::() == 0x444); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_179c) == 0x179c); + static_assert!(offset_of!(HwDataBG15V14_8_3, unit_mask_a) == 0x17c0); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_17e8) == 0x17e8); + static_assert!(offset_of!(HwDataBG15V14_8_3, timer_offset) == 0x17ec); + static_assert!(offset_of!(HwDataBG15V14_8_3, all_ones_masks) == 0x1820); + static_assert!(offset_of!(HwDataBG15V14_8_3, fstp_override) == 0x1850); + static_assert!(offset_of!(HwDataBG15V14_8_3, unk_1860) == 0x1860); + static_assert!(size_of::() == 0x20); + + // HwDataA: 0x4360 bytes, G13 13.5 layout, +0x10 from 0x1288 + static_assert!(size_of::() == 0x4360); + static_assert!(offset_of!(HwDataAG15V14_8_3, clocks_per_period) == 0x4); + static_assert!(offset_of!(HwDataAG15V14_8_3, pwr_status) == 0x10); + static_assert!(offset_of!(HwDataAG15V14_8_3, actual_pstate) == 0x2c); + static_assert!(offset_of!(HwDataAG15V14_8_3, min_pstate_scaled) == 0x54); + static_assert!(offset_of!(HwDataAG15V14_8_3, sram_k) == 0x80); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_64c) == 0x658); + static_assert!(offset_of!(HwDataAG15V14_8_3, pwr_filter_a_neg) == 0x664); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_power_2) == 0x6a4); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_tgt_utilization) == 0x788); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_filter_drop_threshold) == 0x7a8); + static_assert!(offset_of!(HwDataAG15V14_8_3, perf_tgt_utilization_2) == 0x7ec); + static_assert!(offset_of!(HwDataAG15V14_8_3, min_pstate_scaled_4) == 0x860); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask) == 0x8ac); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_release_temp_cc) == 0x8b4); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_ki_dt) == 0x8d8); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_kp) == 0x8e8); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_prop_tgt_delta) == 0x900); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_10) == 0x938); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_coef_a1) == 0x970); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_11) == 0xbe0); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_c2c) == 0xe80); + static_assert!(offset_of!(HwDataAG15V14_8_3, power_zone_count) == 0xe84); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_target_filter_a_neg) == 0xe9c); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_filter_tc_periods_x4) == 0xf7c); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_kp) == 0xfa8); + static_assert!(offset_of!(HwDataAG15V14_8_3, avg_power_filter_tc_ms) == 0xfd0); + static_assert!(offset_of!(HwDataAG15V14_8_3, max_pstate_scaled_14) == 0xff8); + // The builder relies on this base for the G15 +0x10f4/+0x11d0 stores + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_e10_0) == 0x10b4); + static_assert!(size_of::() == 0x1b8); + static_assert!(offset_of!(HwDataA140Extra, unk_40) == 0x40); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_filter_a_neg) == 0x4c); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_kp) == 0x6c); + static_assert!(offset_of!(HwDataA140Extra, max_pstate_scaled_1) == 0x7c); + static_assert!(offset_of!(HwDataA140Extra, se_target) == 0x8c); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_filter_time_constant_clks) == 0x9c); + static_assert!(offset_of!(HwDataA140Extra, unk_b0) == 0xb0); + static_assert!(offset_of!(HwDataA140Extra, unk_d8) == 0xd8); + static_assert!(offset_of!(HwDataA140Extra, gpu_se_reset_criteria) == 0xf0); + static_assert!(offset_of!(HwDataA140Extra, unk_11c) == 0x11c); + static_assert!(offset_of!(HwDataA140Extra, max_pstate_scaled_2) == 0x130); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask_2) == 0x1288); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_e28) == 0x1294); + static_assert!(offset_of!(HwDataAG15V14_8_3, fast_die0_sensor_mask_alt) == 0x1a98); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_3640) == 0x3aa4); + static_assert!(offset_of!(HwDataAG15V14_8_3, init_timestamp) == 0x424c); + static_assert!(offset_of!(HwDataAG15V14_8_3, cluster_tables) == 0x42a4); + static_assert!(offset_of!(HwDataAG15V14_8_3, unk_4340) == 0x4340); + static_assert!(offset_of!(HwDataAG15V14_8_3, gpu_keepalive_mode) == 0x4358); + + // E2 StatsFrag: 0x1248 bytes, -1 fields at +0xc18/+0xc30 + static_assert!(size_of::() == 0x1248); + static_assert!( + offset_of!(GpuGlobalStatsFragG15V14_8_3, stats) + + offset_of!(GpuStatsFragG15V14_8_3, cur_stamp_id) + == 0xc18 + ); + static_assert!( + offset_of!(GpuGlobalStatsFragG15V14_8_3, stats) + + offset_of!(GpuStatsFragG15V14_8_3, unk_id) + == 0xc30 + ); + + // G13/G14X 13.5 sizes must not change (regression guard for the per-field gating). + // RuntimePointers: gpu_scratch at 0x280 (0x270 on G14X) + RuntimeScratch; note that these + // are 0x20 more than ver_info word 0 (0x6ba0/0xb390). The others are the current G13/G14X + // sizes. + static_assert!(size_of::>() == 0x6bc0); + static_assert!(size_of::>() == 0xb3b0); + static_assert!(size_of::() == 0x12394); + static_assert!(size_of::() == 0x1715c); + static_assert!(size_of::() == 0x421c); + static_assert!(size_of::() == 0x6c34); + static_assert!(size_of::() == 0x1884); + static_assert!(size_of::() == 0x1884); + static_assert!(size_of::>() == 0xbc); + static_assert!(size_of::>() == 0xbc); + static_assert!(size_of::>() == 0xb4); + static_assert!(size_of::() == 0x30); + static_assert!(size_of::() == 0x1124); + static_assert!(size_of::() == 0x1a34); + static_assert!(size_of::>() == 0x30); +} diff --git a/drivers/gpu/drm/asahi/m3/hw.rs b/drivers/gpu/drm/asahi/m3/hw.rs new file mode 100644 index 00000000000000..c90156bd1244bd --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw.rs @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-SoC hardware configuration structures +//! +//! This module contains the definitions used to store per-GPU and per-SoC configuration data. + +use crate::fw::types::*; +pub(crate) use crate::identity::{GpuGen, GpuHalGeneration, GpuVariant}; +use kernel::c_str; +use kernel::prelude::*; + +const MAX_POWERZONES: usize = 5; + +pub(crate) mod agx3; +pub(crate) mod t600x; +pub(crate) mod t602x; +pub(crate) mod t6030; +pub(crate) mod t6031; +pub(crate) mod t8103; +pub(crate) mod t8112; +pub(crate) mod t8122; +pub(crate) mod t8140; + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + /// The chip revision T6031 (M3 Max) reports (/arm-io chip-revision 0x12). Only the T6031 + /// table names it; no identification path maps a register read to it yet. + #[allow(dead_code)] + B2 = 0x12, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU core type enumeration. Note: Part of the firmware ABI. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum GpuCore { + // Unknown = 0, + // G5P = 1, + // G5G = 2, + // G9P = 3, + // G9G = 4, + // G10P = 5, + // G11P = 6, + // G11M = 7, + // G11G = 8, + // G12P = 9, + // G13P = 10, + G13G = 11, + G13S = 12, + G13C = 13, + // G14P = 14, + G14G = 15, + G14S = 16, + G14C = 17, + G14D = 18, // Split out, unlike G13D + // G15 firmware core type IDs (G15S = 0x17). + // G15M = 19, + // G15P_AGX2 = 20, + // G15P = 21, + G15G = 22, + G15S = 23, + /// Not confirmed on hardware yet; only the T6031 table uses it. + #[allow(dead_code)] + G15C = 24, +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// A single performance state of the GPU. +#[derive(Debug)] +pub(crate) struct PState { + /// Voltage in millivolts, per GPU cluster. + pub(crate) volt_mv: KVec, + /// Frequency in hertz. + pub(crate) freq_hz: u32, + /// Maximum power consumption of the GPU at this pstate, in milliwatts. + pub(crate) pwr_mw: u32, +} + +impl PState { + pub(crate) fn max_volt_mv(&self) -> u32 { + *self.volt_mv.iter().max().expect("No voltages") + } +} + +/// A power zone definition (we have no idea what this is but Apple puts them in the DT). +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct PowerZone { + pub(crate) target: u32, + pub(crate) target_offset: u32, + pub(crate) filter_tc: u32, +} + +/// An MMIO mapping used by the firmware. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IOMapping { + /// Base physical address of the mapping. + pub(crate) base: usize, + /// Whether this mapping should be replicated to all dies + pub(crate) per_die: bool, + /// Number of mappings. + pub(crate) count: usize, + /// Size of one mapping. + pub(crate) size: usize, + /// Stride between mappings. + pub(crate) stride: usize, + /// Whether the mapping should be writable. + pub(crate) writable: bool, +} + +impl IOMapping { + /// Convenience constructor for a new IOMapping. + pub(crate) const fn new( + base: usize, + per_die: bool, + count: usize, + size: usize, + stride: usize, + writable: bool, + ) -> IOMapping { + IOMapping { + base, + per_die, + count, + size, + stride, + writable, + } + } +} + +/// Unknown HwConfigA fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigA { + pub(crate) unk_87c: i32, + pub(crate) unk_8cc: u32, + pub(crate) unk_e24: u32, +} + +/// Unknown HwConfigB fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigB { + pub(crate) unk_454: u32, + pub(crate) unk_4e0: u64, + pub(crate) unk_534: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: u32, + pub(crate) unk_b30: u32, +} + +/// Render command configs that vary from SoC to SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwRenderConfig { + /// Vertex/tiling-related configuration register (lsb: disable clustering) + pub(crate) tiling_control: u32, +} + +#[derive(Debug)] +pub(crate) struct HwConfigShared2Curves { + pub(crate) t1_coef: u32, + pub(crate) t2: &'static [i16], + pub(crate) t3_coefs: &'static [u32], + pub(crate) t3_scales: &'static [u32], +} + +/// Static hardware clustering configuration for multi-cluster SoCs. +#[derive(Debug)] +pub(crate) struct HwClusteringConfig { + pub(crate) meta1_blocksize: usize, + pub(crate) meta2_size: usize, + pub(crate) meta3_size: usize, + pub(crate) meta4_size: usize, + pub(crate) max_splits: usize, +} + +/// Static hardware configuration for a given SoC model. +#[derive(Debug)] +pub(crate) struct HwConfig { + /// Chip ID in hex format (e.g. 0x8103 for t8103). + pub(crate) chip_id: u32, + /// GPU generation. + pub(crate) gpu_gen: GpuGen, + /// GPU variant type. + pub(crate) gpu_variant: GpuVariant, + /// GPU core type ID (as known by the firmware). + /// + /// This is a firmware-ABI value (renumbered per firmware build). It is + /// `None` where no firmware GpuCore id has been recovered (AGX3 parts): + /// consumers must fail closed rather than substitute a number. + pub(crate) gpu_core: Option, + + /// Base clock used used for timekeeping. + pub(crate) base_clock_hz: u32, + /// Output address space for the UAT on this SoC. + pub(crate) uat_oas: u32, + /// Input-address bits translated by each UAT root. + pub(crate) uat_ias: u8, + /// Number of dies on this SoC. + pub(crate) num_dies: u32, + /// Maximum number of clusters on this SoC. + pub(crate) max_num_clusters: u32, + /// Maximum number of cores per cluster for this GPU. + pub(crate) max_num_cores: u32, + /// Maximum number of frags per cluster for this GPU. + pub(crate) max_num_frags: u32, + /// Maximum number of GPs per cluster for this GPU. + pub(crate) max_num_gps: u32, + + /// Required size of the first preemption buffer. + pub(crate) preempt1_size: usize, + /// Required size of the second preemption buffer. + pub(crate) preempt2_size: usize, + /// Required size of the third preemption buffer. + pub(crate) preempt3_size: usize, + + /// Required size of the compute preemption buffer. + pub(crate) compute_preempt1_size: usize, + + pub(crate) clustering: Option, + + /// Rendering-relevant configuration. + pub(crate) render: HwRenderConfig, + + /// Misc HWDataA field values. + pub(crate) da: HwConfigA, + /// Misc HWDataB field values. + pub(crate) db: HwConfigB, + /// HwDataShared1.table. + pub(crate) shared1_tab: &'static [i32], + /// HwDataShared1.unk_a4. + pub(crate) shared1_a4: u32, + /// HwDataShared2.table. + pub(crate) shared2_tab: &'static [i32], + /// HwDataShared2.unk_508. + pub(crate) shared2_unk_508: u32, + /// HwDataShared2.unk_508. + pub(crate) shared2_curves: Option, + + /// HwDataShared3.unk_8. + pub(crate) shared3_unk: u32, + /// HwDataShared3.table. + pub(crate) shared3_tab: &'static [u32], + + /// Globals.idle_off_standby_timer. + pub(crate) idle_off_standby_timer_default: u32, + /// Globals.unk_hws2_4. + pub(crate) unk_hws2_4: Option<[F32; 8]>, + /// Globals.unk_hws2_24. + pub(crate) unk_hws2_24: u32, + /// Globals.unk_54 + pub(crate) global_unk_54: u16, + + /// Constant related to SRAM voltages. + pub(crate) sram_k: F32, + /// Unknown per-cluster coefficients 1. + pub(crate) unk_coef_a: &'static [&'static [F32]], + /// Unknown per-cluster coefficients 2. + pub(crate) unk_coef_b: &'static [&'static [F32]], + /// Unknown table in Global struct. + pub(crate) global_tab: Option<&'static [u8]>, + /// Whether this GPU has CS/AFR performance states + pub(crate) has_csafr: bool, + + /// Temperature sensor list (8 bits per sensor). + pub(crate) fast_sensor_mask: [u64; 2], + /// Temperature sensor list (alternate). + pub(crate) fast_sensor_mask_alt: [u64; 2], + /// Temperature sensor present bitmask. + pub(crate) fast_die0_sensor_present: u32, + /// Required MMIO mappings for this GPU/firmware. + pub(crate) io_mappings: &'static [Option], + /// SRAM base + pub(crate) sram_base: Option, + /// SRAM size + pub(crate) sram_size: Option, +} + +/// Dynamic (fetched from hardware/DT) configuration. +#[derive(Debug)] +pub(crate) struct DynConfig { + /// Base physical address of the UAT TTB (from DT reserved memory region). + pub(crate) uat_ttb_base: u64, + /// GPU ID configuration read from hardware. + pub(crate) id: GpuIdConfig, + /// Power calibration configuration for this specific chip/device. + pub(crate) pwr: PwrConfig, + /// Firmware version. + #[allow(dead_code)] + pub(crate) firmware_version: KVec, + + pub(crate) hw_data_a: KVVec, + pub(crate) hw_data_b: KVVec, + pub(crate) hw_globals: KVVec, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// Unified shader core generation. + pub(crate) usc_generation: u32, + /// Apple GPU HAL generation (0 for the pre-numbered generations). + pub(crate) gpu_hal_generation: GpuHalGeneration, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Number of GPU dies read from the ID register. + pub(crate) num_dies: u32, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: KVec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: KVec, + +} + +/// Configurable CS/AFR GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct CsAfrPwrConfig { + /// GPU CS performance state list. + pub(crate) perf_states_cs: KVec, + /// GPU AFR performance state list. + pub(crate) perf_states_afr: KVec, + + /// CS leakage coefficient per die. + pub(crate) leak_coef_cs: KVec, + /// AFR leakage coefficient per die. + pub(crate) leak_coef_afr: KVec, + + /// Minimum voltage for the CS/AFR SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, +} + +/// Configurable GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct PwrConfig { + /// GPU performance state list. + pub(crate) perf_states: KVec, + /// GPU power zone list. + pub(crate) power_zones: KVec, + + /// Core leakage coefficient per cluster. + pub(crate) core_leak_coef: KVec, + /// SRAM leakage coefficient per cluster. + pub(crate) sram_leak_coef: KVec, + + pub(crate) csafr: Option, + + /// Maximum total power of the GPU in milliwatts. + pub(crate) max_power_mw: u32, + /// Maximum frequency of the GPU in megahertz. + pub(crate) max_freq_mhz: u32, + + /// Minimum performance state to start at. + pub(crate) perf_base_pstate: u32, + /// Maximum enabled performance state. + pub(crate) perf_max_pstate: u32, + + /// Minimum voltage for the SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, + + // Most of these fields are just named after Apple ADT property names and we don't fully + // understand them. They configure various power-related PID loops and filters. + /// Average power filter time constant in milliseconds. + pub(crate) avg_power_filter_tc_ms: u32, + /// Average power filter PID integral gain? + pub(crate) avg_power_ki_only: F32, + /// Average power filter PID proportional gain? + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + /// Average power target filter time constant in periods. + pub(crate) avg_power_target_filter_tc: u32, + /// "Fast die0" (temperature?) PID integral gain. + pub(crate) fast_die0_integral_gain: F32, + /// "Fast die0" (temperature?) PID proportional gain. + pub(crate) fast_die0_proportional_gain: F32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_release_temp: u32, + /// Delay from the fender (?) becoming idle to powerdown + pub(crate) fender_idle_off_delay_ms: u32, + /// Timeout from firmware early wake to sleep if no work was submitted (?) + pub(crate) fw_early_wake_timeout_ms: u32, + /// Delay from the GPU becoming idle to powerdown + pub(crate) idle_off_delay_ms: u32, + /// Related to the above? + pub(crate) idle_off_standby_timer: u32, + /// Percent? + pub(crate) perf_boost_ce_step: u32, + /// Minimum utilization before performance state is increased in %. + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_filter_drop_threshold: u32, + /// Performance PID filter time constant? (periods?) + pub(crate) perf_filter_time_constant: u32, + /// Performance PID filter time constant 2? (periods?) + pub(crate) perf_filter_time_constant2: u32, + /// Performance PID integral gain. + pub(crate) perf_integral_gain: F32, + /// Performance PID integral gain 2 (?). + pub(crate) perf_integral_gain2: F32, + pub(crate) perf_integral_min_clamp: u32, + /// Performance PID proportional gain. + pub(crate) perf_proportional_gain: F32, + /// Performance PID proportional gain 2 (?). + pub(crate) perf_proportional_gain2: F32, + pub(crate) perf_reset_iters: u32, + /// Target GPU utilization for the performance controller in %. + pub(crate) perf_tgt_utilization: u32, + /// Power sampling period in milliseconds. + pub(crate) power_sample_period: u32, + /// PPM (?) filter time constant in milliseconds. + pub(crate) ppm_filter_time_constant_ms: u32, + /// PPM (?) filter PID integral gain. + pub(crate) ppm_ki: F32, + /// PPM (?) filter PID proportional gain. + pub(crate) ppm_kp: F32, + /// Power consumption filter time constant (periods?) + pub(crate) pwr_filter_time_constant: u32, + /// Power consumption filter PID integral gain. + pub(crate) pwr_integral_gain: F32, + pub(crate) pwr_integral_min_clamp: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) pwr_proportional_gain: F32, + /// Power sample period in base clocks, used when not an integer number of ms + pub(crate) pwr_sample_period_aic_clks: u32, + + pub(crate) se_engagement_criteria: i32, + pub(crate) se_filter_time_constant: u32, + pub(crate) se_filter_time_constant_1: u32, + pub(crate) se_inactive_threshold: u32, + pub(crate) se_ki: F32, + pub(crate) se_ki_1: F32, + pub(crate) se_kp: F32, + pub(crate) se_kp_1: F32, + pub(crate) se_reset_criteria: u32, + /// Shader-engine controller target (G15 runtime layout). + pub(crate) se_target: u32, +} + +impl PwrConfig { + fn load_opp( + dev: &kernel::device::Device, + name: &CStr, + cfg: &HwConfig, + is_main: bool, + ) -> Result> { + let mut perf_states = KVec::new(); + + let node = dev.of_node().ok_or(EIO)?; + let opps = node.parse_phandle(name, 0).ok_or(EIO)?; + + for opp in opps.children() { + let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let mut volt_uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + let pwr_uw: u32 = if is_main { + opp.get_property(c_str!("opp-microwatt"))? + } else { + 0 + }; + + let voltage_count = if is_main { + cfg.max_num_clusters + } else { + cfg.num_dies + }; + + if volt_uv.len() != voltage_count as usize { + dev_err!( + dev, + "Invalid opp-microvolt length (expected {}, got {})\n", + voltage_count, + volt_uv.len() + ); + return Err(EINVAL); + } + + // Round up: a voltage is never published below the device tree's. + volt_uv.iter_mut().for_each(|a| *a = if cfg.gpu_gen == GpuGen::G15 { a.div_ceil(1000) } else { *a / 1000 }); + let volt_mv = volt_uv; + + let pwr_mw = pwr_uw / 1000; + + perf_states.push( + PState { + freq_hz: freq_hz.try_into()?, + volt_mv, + pwr_mw, + }, + GFP_KERNEL, + )?; + } + + if perf_states.is_empty() { + Err(EINVAL) + } else { + Ok(perf_states) + } + } + + /// Load the GPU power configuration from the device tree node of `dev`. + pub(crate) fn load(dev: &kernel::device::Device, cfg: &HwConfig) -> Result { + let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?; + let node = dev.of_node().ok_or(EIO)?; + + macro_rules! prop { + ($prop:expr, $default:expr) => {{ + node.get_opt_property(c_str!($prop)) + .map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + .unwrap_or($default) + }}; + ($prop:expr) => {{ + node.get_property(c_str!($prop)).map_err(|e| { + dev_err!(dev, "Error reading property {}: {:?}\n", $prop, e); + e + })? + }}; + } + + let pz_data = prop!("apple,power-zones", KVec::new()); + + if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { + dev_err!(dev, "Invalid apple,power-zones value\n"); + return Err(EINVAL); + } + + let pz_count = pz_data.len() / 3; + let mut power_zones = KVec::new(); + for i in (0..pz_count).step_by(3) { + power_zones.push( + PowerZone { + target: pz_data[i], + target_offset: pz_data[i + 1], + filter_tc: pz_data[i + 2], + }, + GFP_KERNEL, + )?; + } + + let core_leak_coef: KVec = prop!("apple,core-leak-coef"); + let sram_leak_coef: KVec = prop!("apple,sram-leak-coef"); + + if core_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev, "Invalid apple,core-leak-coef\n"); + return Err(EINVAL); + } + if sram_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev, "Invalid apple,sram_leak_coef\n"); + return Err(EINVAL); + } + + let csafr = if cfg.has_csafr { + Some(CsAfrPwrConfig { + perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?, + perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?, + leak_coef_cs: prop!("apple,cs-leak-coef"), + leak_coef_afr: prop!("apple,afr-leak-coef"), + min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"), + }) + } else { + None + }; + + let power_sample_period: u32 = prop!("apple,power-sample-period"); + + Ok(PwrConfig { + core_leak_coef, + sram_leak_coef, + + max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(), + max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000, + + perf_base_pstate: prop!("apple,perf-base-pstate", 1), + perf_max_pstate: perf_states.len() as u32 - 1, + min_sram_microvolt: if cfg.gpu_gen == GpuGen::G15 { + // SRAM rail floor: 810 mV from the J516S runtime ADT perf-states-sram table. + // Tolerate a DT without the property on G15 only. + prop!("apple,min-sram-microvolt", t6030::MIN_SRAM_MICROVOLT) + } else { + prop!("apple,min-sram-microvolt") + }, + + avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"), + avg_power_ki_only: prop!("apple,avg-power-ki-only"), + avg_power_kp: prop!("apple,avg-power-kp"), + avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"), + avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"), + fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"), + fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"), + fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0), + fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80), + fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40), + fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5), + idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2), + idle_off_standby_timer: prop!( + "apple,idleoff-standby-timer", + cfg.idle_off_standby_timer_default + ), + perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25), + perf_boost_min_util: prop!("apple,perf-boost-min-util", 100), + perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"), + perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"), + perf_filter_time_constant: prop!("apple,perf-filter-time-constant"), + perf_integral_gain2: prop!("apple,perf-integral-gain2"), + perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)), + perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"), + perf_proportional_gain2: prop!("apple,perf-proportional-gain2"), + perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), + perf_reset_iters: prop!("apple,perf-reset-iters", 6), + perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), + power_sample_period, + ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), + ppm_ki: prop!("apple,ppm-ki"), + ppm_kp: prop!("apple,ppm-kp"), + pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313), + pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)), + pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), + pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), + pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), + pwr_sample_period_aic_clks: prop!( + "apple,pwr-sample-period-aic-clks", + cfg.base_clock_hz / 1000 * power_sample_period + ), + se_engagement_criteria: prop!("apple,se-engagement-criteria", -1), + se_filter_time_constant: prop!("apple,se-filter-time-constant", 9), + se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3), + se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500), + se_ki: prop!("apple,se-ki", f32!(-50.0)), + se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)), + se_kp: prop!("apple,se-kp", f32!(-5.0)), + se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)), + se_reset_criteria: prop!("apple,se-reset-criteria", 50), + se_target: if cfg.gpu_gen == GpuGen::G15 { prop!("apple,se-target", 1400) } else { 1400 }, + + perf_states, + power_zones, + csafr, + }) + } + + pub(crate) fn max_frequency_khz(&self) -> u32 { + self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000 + } +} diff --git a/drivers/gpu/drm/asahi/m3/hw/agx3.rs b/drivers/gpu/drm/asahi/m3/hw/agx3.rs new file mode 100644 index 00000000000000..cb0c4fdbfa0126 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw/agx3.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +#[cfg(not(test))] +use crate::identity::{GpuGen, GpuVariant}; +#[cfg(test)] +#[path = "../identity.rs"] +mod identity; +#[cfg(test)] +use identity::{GpuGen, GpuVariant}; + +/// Identification-only configuration for one AGX3 SoC. +#[derive(Debug)] +pub(crate) struct SocConfig { + /// Chip ID in hex format (e.g. 0x6050 for t6050). + pub(crate) chip_id: u32, + /// Decoded design generation for this chip. + pub(crate) gpu_gen: GpuGen, + /// Decoded variant for this chip. + pub(crate) gpu_variant: GpuVariant, + /// Hardware family byte expected at `0xd04000[31:24]`. + pub(crate) hw_family: u8, + /// Hardware variant byte expected at `0xd04000[23:16]`. + pub(crate) hw_variant: u8, + /// Die count expected at `0xd04010[19:16]`. + pub(crate) num_dies: u32, + /// Input-address bits per UAT root (42 on every AGX3 part). + pub(crate) uat_ias: u8, + /// Output address space (42-bit on every AGX3 part). + pub(crate) uat_oas: u32, +} + +const fn agx3( + chip_id: u32, + gpu_gen: GpuGen, + gpu_variant: GpuVariant, + hw_family: u8, + hw_variant: u8, + num_dies: u32, +) -> SocConfig { + SocConfig { + chip_id, + gpu_gen, + gpu_variant, + hw_family, + hw_variant, + num_dies, + uat_ias: 42, + uat_oas: 42, + } +} + +pub(crate) const T8122: SocConfig = agx3(0x8122, GpuGen::G15, GpuVariant::G, 0x7, 2, 1); +/// Apple M3 Pro (G15S). +pub(crate) const T6030: SocConfig = agx3(0x6030, GpuGen::G15, GpuVariant::S, 0x7, 3, 1); +/// Apple M3 Max (G15C). +pub(crate) const T6031: SocConfig = agx3(0x6031, GpuGen::G15, GpuVariant::C, 0x7, 4, 1); +/// Apple M3 Ultra (G15D, two dies). +pub(crate) const T6032: SocConfig = agx3(0x6032, GpuGen::G15, GpuVariant::D, 0x7, 4, 2); diff --git a/drivers/gpu/drm/asahi/m3/hw/t8122.rs b/drivers/gpu/drm/asahi/m3/hw/t8122.rs new file mode 100644 index 00000000000000..5ba6b0ef6acd6f --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/hw/t8122.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for T8122 (M3, G15G): one die, one cluster of ten core slots. +//! +//! Only the M3 runtime reads this table (`m3_soc::T8122`), and it refuses a T8122 until the +//! rest of the SoC's runtime configuration is known (`m3_soc::Soc::require_complete`). The +//! G15 manager backend never starts on T8122. + +use crate::f32; + +use super::*; + +/// Fender: ADT sgx reg[1] translated through the /arm-io ranges (SGX + 0xd00000). +const FENDER_BASE: usize = 0x2_90d0_0000; +/// The GPU register window: ADT sgx reg[0] translated through the /arm-io ranges. +const SGX_BASE: usize = 0x2_9000_0000; + +/// The firmware IO mappings, one per HwDataB slot (31 on G15). A slot is `None` when T8122 has +/// no block for it; its record then stays zero. Every address is a CPU physical address. +const IO_MAPPINGS: [Option; 31] = [ + // 0: Fender. The window is 0x104000 bytes: the ADT sgx reg[1] size (0x12c000) less 0x28000, + // the same difference as on T6030. + Some(IOMapping::new(FENDER_BASE, false, 1, 0x10_4000, 0, true)), + // 1: AIC timer, the same 16 KiB page on every SoC. + Some(IOMapping::new(0x2_0e10_0000, false, 1, 0x4000, 0, false)), + // 2: AIC software interrupts: AIC (0x2d1000000) + 0x14000, the page that holds the ADT's + // meta-sw-interrupt registers. + Some(IOMapping::new(0x2_d101_4000, false, 1, 0x4000, 0, true)), + // 3: RGX: the first 128 KiB of the GPU register window. + Some(IOMapping::new(SGX_BASE, false, 1, 0x2_0000, 0, true)), + None, // 4: UVD: the ADT sgx node has no third reg range + None, // 5: unused + None, // 6: display underrun workaround + None, // 7: analog temperature sensors + None, // 8: PMP doorbell + // 9: metrology sensors, SGX + 0xe08000. + Some(IOMapping::new(SGX_BASE + 0xe0_8000, false, 1, 0x8000, 0, true)), + // 10: GM GIFAF registers, a 4 KiB page at Fender + 0xd000. + Some(IOMapping::new(FENDER_BASE + 0xd000, false, 1, 0x1000, 0, true)), + // 11: memory cache, two instances of 0x58000 bytes (a multiple of 16 KiB), at 0x220000000 + // and 0x222000000. + Some(IOMapping::new(0x2_2000_0000, false, 2, 0x5_8000, 0x200_0000, true)), + None, // 12: AIC banked registers + None, // 13: PMGR scratch + None, // 14: NIA idle register, die 0 + None, // 15: NIA idle register, die 1 + None, // 16: CRE registers + None, // 17: streaming codec registers + // 18: telemetry dashboard, written by the firmware: the 4 KiB at +0x10000 of the PMP's + // telemetry window (0x2d03c0000, 0x14000 bytes). + Some(IOMapping::new(0x2_d03d_0000, false, 1, 0x1000, 0, true)), + // 19: telemetry dashboard, read by the firmware: the first 8 KiB of the same window. + Some(IOMapping::new(0x2_d03c_0000, false, 1, 0x2000, 0, false)), + None, // 20: telemetry dashboard configuration + None, // 21 + None, // 22 + None, // 23: AFR registers + None, // 24 + // 25: ANE doorbell, ANE + 0x45c000. + Some(IOMapping::new(0x3_1145_c000, false, 1, 0x4000, 0, true)), + // 26: PMS metrology sensors. + Some(IOMapping::new(0x2_d028_0000, false, 1, 0x8000, 0, false)), + None, // 27 + None, // 28 + // 29: GPU clock generator, SGX + 0xe1c000. + Some(IOMapping::new(SGX_BASE + 0xe1_c000, false, 1, 0x4000, 0, false)), + None, // 30 +]; + +/// T8122 (M3, G15G): one die, one cluster of ten core slots, eight or ten of them active. +pub(crate) const HWCONFIG_T8122: super::HwConfig = HwConfig { + chip_id: 0x8122, + gpu_gen: GpuGen::G15, // ID_VERSION[31:24] == 7 + gpu_variant: GpuVariant::G, // ID_VERSION[23:16] == 2 + gpu_core: Some(GpuCore::G15G), + + // The 24 MHz reference clock (HwDataB base_clock_khz = 24000). + base_clock_hz: 24_000_000, + // 42-bit UAT roots and output addresses, as on every AGX3 part (hw::agx3::T8122). + uat_ias: 42, + uat_oas: 42, + num_dies: 1, // ID_COUNTS_1[19:16] + max_num_clusters: 1, // one cluster of ten core slots + max_num_cores: 10, + max_num_frags: 10, // one fragment unit per core slot + max_num_gps: 4, // ID_COUNTS_2[23:16] + + // Not read by the M3 runtime, which keeps its own buffer layout. + preempt1_size: 0, + preempt2_size: 0, + preempt3_size: 0, + // Compute preemption buffer: 0xd80 bytes plus 0x700 per cluster. + compute_preempt1_size: 0x1480, + clustering: None, + render: HwRenderConfig { tiling_control: 0 }, + + da: HwConfigA { + // Cleared by the M3 runtime's InitData in any case. + unk_87c: 0, + // Fast-die temperature target, in centidegrees Celsius (110 C, as on T6030). The J613 + // ADT's gpu-fast-die0-target is 88; how the two relate is not known. + unk_8cc: 11_000, + unk_e24: 0, + }, + db: HwConfigB { + // The chip revision's major number: /arm-io chip-revision 0x20 >> 4. + unk_454: 2, + // The G15 InitData builders read none of the words below. + unk_4e0: 0, + unk_534: 0, + unk_ab8: 0, + unk_abc: 0, + unk_b30: 1, + }, + // G15 leaves the hws1/hws2/hws3 curves zero, as on T6030. + shared1_tab: &[], + shared1_a4: 0, + shared2_tab: &[], + shared2_unk_508: 0, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + // Idle-off standby timer when the device tree gives none: the J613 ADT has no + // gpu-idleoff-standby-timer. + idle_off_standby_timer_default: 700, + // The smart idle-off curve and count, the same as T6030's. + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])), + unk_hws2_24: 6, + // The M3 runtime writes 0xffff to this Globals word in any case. + global_unk_54: 0, + // SRAM power scale 1.02 for every performance state, as on T6030. + sram_k: f32!(1.02), + unk_coef_a: &[], + unk_coef_b: &[], + // No FwUtil performance tunables are written. + global_tab: None, + has_csafr: false, + // The fast-die sensor mask is each machine's: the M3 runtime takes it from the boot + // loader's apple,fast-die0-sensor-mask (the ADT's gpu-fast-die0-sensor-mask), not from here. + fast_sensor_mask: [0, 0], + fast_sensor_mask_alt: [0, 0], + fast_die0_sensor_present: 0, + io_mappings: &IO_MAPPINGS, + // No Fender scratch SRAM window. + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/m3/identity.rs b/drivers/gpu/drm/asahi/m3/identity.rs new file mode 100644 index 00000000000000..f468272c5680ed --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/identity.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +/// Supported GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, + G15 = 15, + G17 = 17, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuHalGeneration { + Legacy = 0, + Hal200 = 200, +} + +/// Public architecture axes for the G15 ID-register decoder. +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub(crate) struct GpuIdentity { + pub(crate) gpu_gen: GpuGen, + pub(crate) gpu_variant: GpuVariant, + pub(crate) usc_generation: u32, + pub(crate) gpu_hal_generation: GpuHalGeneration, +} + +pub(crate) fn decode_gpu_identity(family: u8, variant: u8, num_dies: u8) -> Option { + use GpuVariant::*; + if family != 7 { return None; } + let (gpu_variant, usc_generation) = match variant { + 0 => (P, 2), + 2 => (G, 3), + 3 => (S, 3), + 4 => match num_dies { 1 => (C, 3), 2 => (D, 3), _ => return None }, + _ => return None, + }; + Some(GpuIdentity { gpu_gen: GpuGen::G15, gpu_variant, usc_generation, + gpu_hal_generation: GpuHalGeneration::Legacy }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn g15_variants_keep_their_public_identity() { + assert_eq!(decode_gpu_identity(7, 3, 1).unwrap().gpu_variant, GpuVariant::S); + assert_eq!(decode_gpu_identity(7, 4, 1).unwrap().gpu_variant, GpuVariant::C); + assert_eq!(decode_gpu_identity(7, 4, 2).unwrap().gpu_variant, GpuVariant::D); + assert_eq!(decode_gpu_identity(7, 2, 1).unwrap().usc_generation, 3); + assert!(decode_gpu_identity(7, 4, 0).is_none()); + assert!(decode_gpu_identity(10, 2, 1).is_none()); + } +} diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs new file mode 100644 index 00000000000000..dfeffd900eb12b --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -0,0 +1,1457 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! GPU initialization data builder. +//! +//! The root of all interaction between the GPU firmware and the host driver is a complex set of +//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration +//! and the pointers to the ring buffers and global data fields that are used for communication at +//! runtime. +//! +//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values +//! derived from observing the InitData structures that macOS generates. + +use crate::f32; +use crate::fw::initdata::*; +use crate::fw::types::*; +use crate::module_parameters; +use crate::{ + driver::AsahiDevice, + gem, + gpu, + hw, + mmu, // +}; +use kernel::error::{ + Error, + Result, // +}; +use kernel::macros::versions; +use kernel::prelude::*; +use kernel::try_init; + +use ::pin_init; +use ::pin_init::Init; + +/// Builder helper for the global GPU InitData. + +/// G15 idle-off delay (ms, Globals +0x9a0/+0x9a4) used as "never idle off" while the rings live +/// in Fender scratch SRAM without a backup/restore path. +/// TODO: the firmware's encoding of "never" is unknown. i32::MAX stays positive if the +/// firmware treats the value as signed, and is never smaller than u32::MAX would be if it +/// converts ms to 24 MHz ticks in 32-bit arithmetic (both wrap to ~179 s); with 64-bit +/// arithmetic it is ~24 days. +#[allow(dead_code)] +pub(crate) const G15_KEEP_POWERED_MS: u32 = i32::MAX as u32; + +/// Which T6030 backend a G15 InitData is built for. The two differ where the manager backend +/// keeps the values it was validated with on J516S. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub(crate) enum G15Layout { + /// The G15 GpuManager backend: rings in Fender SRAM, the GPU never idles off. + Manager, + /// The serialized M3 runtime backend: rings in DRAM with finite idle-off, its own GPU and + /// firmware VA layout, and the values its InitData carries for the power, idle and + /// shader-engine controllers. + Runtime, +} + +/// G15 InitData choices that are not part of the hardware or device-tree configuration. +#[derive(Copy, Clone, Debug)] +pub(crate) struct G15Options { + /// The backend the InitData is built for. + pub(crate) layout: G15Layout, + /// Publish the performance states as two voltage-sorted tables (see [`PStateTables`]). + pub(crate) split_pstates: bool, + /// Highest performance state the firmware may use, as an index into the voltage-sorted + /// primary table (1 = lowest). None publishes the whole table. + pub(crate) cap: Option, + /// Firmware VA of the timestamp arena (HwDataB+0x28); None uses the manager's kernel range. + pub(crate) timestamp_base: Option, + /// Manager layout: write the PPM words and the Globals power targets as the runtime layout + /// does (`asahi.g15_debug` bit 55). The runtime layout always does. + pub(crate) reference_ppm: bool, + /// Runtime layout: the SoC's HwDataB configuration words and unit masks + /// (`m3_soc::Soc::hwdata_b`). None for the manager layout, which does not write them. + pub(crate) runtime_hwdata_b: Option<&'static G15RuntimeHwDataB>, + /// Accept operating points whose clusters have different voltages + /// (`m3_soc::Soc::per_cluster_voltages`); see [`PStateTables`]. + pub(crate) per_cluster_voltages: bool, +} + +/// The HwDataB words of the runtime backend's InitData that are neither its GPU VA layout nor +/// device-tree values: configuration words where the manager layout keeps 0 or its validated +/// values (including the A/B pair +0xa98/+0xb40 at 0), and the two unit masks. They are values +/// of one SoC's runtime InitData (`m3_soc::Soc::hwdata_b`), not derived from the device tree. +#[derive(Debug)] +pub(crate) struct G15RuntimeHwDataB { + pub(crate) unk_454: u32, + pub(crate) unk_464: u32, + pub(crate) unk_a7c: u64, + pub(crate) unk_a98: u32, + pub(crate) unk_abc: u64, + pub(crate) unk_ae4: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: u64, + /// +0xb40, the "UAT enabled" word of the manager layout. + pub(crate) unk_554: u32, + pub(crate) unk_17b8: u32, + /// Unit enable masks, +0x17c0/+0x17c8. + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u32, + pub(crate) unk_1808: u32, + pub(crate) unk_1818: u32, +} + +impl G15Options { + /// The options of the G15 manager backend: its validated defaults, the performance-state + /// cap (`asahi.m3_max_pstate`) and the `asahi.g15_debug` A/B switches. + fn manager(cfg: &'static hw::HwConfig) -> G15Options { + let g15 = cfg.gpu_gen == hw::GpuGen::G15; + G15Options { + layout: G15Layout::Manager, + split_pstates: g15 + && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerSplitPstates), + cap: g15.then(|| { + crate::m3_params::max_pstate_param().unwrap_or(crate::m3_params::MAX_PSTATE_DEFAULT) + }), + timestamp_base: None, + reference_ppm: g15 + && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerReferencePpm), + runtime_hwdata_b: None, + per_cluster_voltages: false, + } + } + + /// The published performance-state tables for `pwr`. + pub(crate) fn tables(&self, pwr: &hw::PwrConfig) -> Result { + PStateTables::new(pwr, self.split_pstates, self.cap, self.per_cluster_voltages) + } + + /// RuntimePointers+0x2d0, a configuration word: 4 in the runtime backend's InitData, 0 (its + /// validated value) for the manager. + pub(crate) fn runtime_pointers_2d0(&self) -> u32 { + match self.layout { + G15Layout::Manager => 0, + G15Layout::Runtime => 4, + } + } +} + +/// Start of the USC (shader code) window in the runtime backend's GPU VA layout. +const RUNTIME_USC_BASE: u64 = 0x10_0000_0000; +/// The runtime backend's unknown-page address, the end of its 42-bit user VA range. +const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; + +/// Maximum number of entries of a firmware performance-state table, including the off state. +const PSTATE_TABLE_ENTRIES: usize = 16; + +/// The performance-state tables as published to the G15 firmware. +/// +/// The device tree lists the performance states in the bootloader's order: several states can +/// share one voltage, and the frequencies are not sorted. The firmware takes two tables sorted +/// by voltage: the primary one with the highest frequency of each voltage, the secondary one with +/// the lowest, plus the device-tree index of every entry. Entry 0 is the off state in both. +/// Every frequency and voltage comes from the device tree; nothing is interpolated. +/// +/// With per-cluster voltages (T6031, whose clusters have their own binned voltage tables), the +/// states are grouped by their whole voltage list instead: states that share their highest +/// cluster voltage must have the same voltage on every cluster, the groups are ordered by that +/// highest voltage, and every cluster's voltage must rise from one entry to the next. +#[derive(Copy, Clone, Debug)] +pub(crate) struct PStateTables { + /// Number of entries, including the off state. + pub(crate) len: usize, + /// Device-tree index of each primary-table entry. + pub(crate) primary: [u8; PSTATE_TABLE_ENTRIES], + /// Device-tree index of each secondary-table entry (split tables only). + pub(crate) secondary: Option<[u8; PSTATE_TABLE_ENTRIES]>, + /// Highest primary-table entry the firmware may select. + pub(crate) max: u32, +} + +impl PStateTables { + /// The primary/secondary split of `states`: one entry per distinct voltage, ascending. + fn split( + states: &[hw::PState], + ) -> Result<([u8; PSTATE_TABLE_ENTRIES], [u8; PSTATE_TABLE_ENTRIES], usize)> { + Self::split_for(states, false) + } + + /// [`Self::split`], accepting per-cluster voltages when `per_cluster` is set. + fn split_for( + states: &[hw::PState], + per_cluster: bool, + ) -> Result<([u8; PSTATE_TABLE_ENTRIES], [u8; PSTATE_TABLE_ENTRIES], usize)> { + let mut primary = [0u8; PSTATE_TABLE_ENTRIES]; + let mut secondary = [0u8; PSTATE_TABLE_ENTRIES]; + if states.len() < 2 || states.len() > u8::MAX as usize || states[0].freq_hz != 0 { + return Err(EINVAL); + } + let mut len = 1; + let mut last_mv = 0; + loop { + // The next voltage above `last_mv`, and its fastest and slowest states. + let mut next: Option<(u32, usize, usize)> = None; + for (i, ps) in states.iter().enumerate().skip(1) { + let mv = ps.max_volt_mv(); + // Every state after the off state runs, with one voltage for all clusters + // unless the SoC gives each cluster its own. + if ps.freq_hz == 0 || (!per_cluster && ps.volt_mv.iter().any(|v| *v != mv)) { + return Err(EINVAL); + } + if mv <= last_mv { + continue; + } + // States of one entry run every cluster at the same voltages. + if let (true, Some((best, hi, _))) = (per_cluster, next) { + if mv == best && states[hi].volt_mv[..] != ps.volt_mv[..] { + return Err(EINVAL); + } + } + next = match next { + Some((best, hi, lo)) if mv == best => Some(( + mv, + if ps.freq_hz > states[hi].freq_hz { i } else { hi }, + if ps.freq_hz < states[lo].freq_hz { i } else { lo }, + )), + Some((best, _, _)) if mv > best => next, + _ => Some((mv, i, i)), + }; + } + let Some((mv, hi, lo)) = next else { + break; + }; + if len >= PSTATE_TABLE_ENTRIES + || states[hi].freq_hz <= states[primary[len - 1] as usize].freq_hz + || states[lo].freq_hz <= states[secondary[len - 1] as usize].freq_hz + { + // Faster states must need a higher voltage in both tables. + return Err(EINVAL); + } + // With per-cluster voltages, every cluster's voltage rises with the entry. + if per_cluster && len > 1 { + let previous = &states[primary[len - 1] as usize].volt_mv; + let current = &states[hi].volt_mv; + if previous.len() != current.len() + || previous.iter().zip(current.iter()).any(|(p, c)| c <= p) + { + return Err(EINVAL); + } + } + primary[len] = hi as u8; + secondary[len] = lo as u8; + len += 1; + last_mv = mv; + } + // At least one running state must have a nonzero voltage; callers + // index the table up to len - 1 and clamp caps to 1..=len - 1. + if len < 2 { + return Err(EINVAL); + } + Ok((primary, secondary, len)) + } + + /// Build the published tables. `cap` is an index into the voltage-sorted primary table, + /// also when the states are published in device-tree order: the firmware may then use any + /// device-tree state up to the capped frequency, and the cap is translated to the highest + /// device-tree index whose states all stay at or below it. + /// `per_cluster` accepts per-cluster voltages (see [`PStateTables`]). + pub(crate) fn new( + pwr: &hw::PwrConfig, + split: bool, + cap: Option, + per_cluster: bool, + ) -> Result { + let states = &pwr.perf_states; + if split { + let (primary, secondary, len) = Self::split_for(states, per_cluster)?; + let top = len as u64 - 1; + let max = cap.map_or(top, |c| c.clamp(1, top)) as u32; + return Ok(PStateTables { + len, + primary, + secondary: Some(secondary), + max, + }); + } + if states.len() > PSTATE_TABLE_ENTRIES { + return Err(EINVAL); + } + let mut primary = [0u8; PSTATE_TABLE_ENTRIES]; + for (i, slot) in primary.iter_mut().enumerate().take(states.len()) { + *slot = i as u8; + } + let top = states.len() as u32 - 1; + let max = match cap { + // No voltage-sorted table can have more entries than the device tree has states, + // so such a cap leaves the whole table. + None => top, + Some(c) if c >= u64::from(top) => top, + Some(c) => { + let (sorted, _, sorted_len) = Self::split(states)?; + let limit_hz = states[sorted[c.clamp(1, sorted_len as u64 - 1) as usize] as usize].freq_hz; + // The longest device-tree prefix that stays at or below the capped frequency. + let mut max = 1; + while max < top && states[max as usize + 1].freq_hz <= limit_hz { + max += 1; + } + max + } + }; + Ok(PStateTables { + len: states.len(), + primary, + secondary: None, + max, + }) + } + + /// The device-tree state of primary-table entry `i`. + pub(crate) fn primary_state<'s>(&self, pwr: &'s hw::PwrConfig, i: usize) -> &'s hw::PState { + &pwr.perf_states[self.primary[i] as usize] + } + + /// The frequency of the highest entry the firmware may select, in kHz. + pub(crate) fn max_frequency_khz(&self, pwr: &hw::PwrConfig) -> u32 { + self.primary_state(pwr, self.max as usize).freq_hz / 1000 + } +} + +/// The highest GPU frequency the G15 manager backend lets the firmware use, in kHz (its +/// performance cap), or None when the GPU is not G15 or the cap cannot be applied. +pub(crate) fn g15_manager_max_frequency_khz( + cfg: &'static hw::HwConfig, + pwr: &hw::PwrConfig, +) -> Option { + if cfg.gpu_gen != hw::GpuGen::G15 { + return None; + } + let tables = G15Options::manager(cfg).tables(pwr).ok()?; + Some(tables.max_frequency_khz(pwr)) +} + +pub(crate) struct InitDataBuilderG15V14_8_3<'a> { + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: G15Options, +} + +impl<'a> InitDataBuilderG15V14_8_3<'a> { + /// Create a new InitData builder + pub(crate) fn new( + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> InitDataBuilderG15V14_8_3<'a> { + InitDataBuilderG15V14_8_3 { + dev, + alloc, + cfg, + dyncfg, + g15: G15Options::manager(cfg), + } + } + + /// Create an unknown T81xx-specific data structure. + fn t81xx_data( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init { + let _perf_max_pstate = dyncfg.pwr.perf_max_pstate; + + pin_init::init_zeroed::().chain(move |_ret| { + match cfg.chip_id { + 0x8103 | 0x8112 => { + } + _ => (), + } + Ok(()) + }) + } + + /// Build the HwDataA contents (mostly power-related configuration) without allocating the + /// object, so that they can fill a firmware object or a plain buffer alike. + pub(crate) fn hwdata_a_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; + let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; + let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); + let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); + let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); + let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; + let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); + let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); + + let base_ps = pwr.perf_base_pstate; + let base_ps_scaled = 100 * base_ps; + let tables = g15.tables(pwr)?; + let max_ps = tables.max; + let runtime = g15.layout == G15Layout::Runtime; + let max_ps_scaled = 100 * max_ps; + let boost_ps_count = if cfg.gpu_gen == hw::GpuGen::G15 { + max_ps.checked_sub(base_ps).ok_or(EINVAL)? + } else { max_ps - base_ps }; + + #[allow(unused_variables)] + let base_clock_khz = cfg.base_clock_hz / 1000; + let v_clocks_per_period = pwr.pwr_sample_period_aic_clks; + + #[allow(unused_variables)] + let clocks_per_period_coarse = cfg.base_clock_hz / 1000 * pwr.power_sample_period; + + Ok( + try_init!(raw::HwDataAG15V14_8_3 { + clocks_per_period: v_clocks_per_period, + clocks_per_period_2: v_clocks_per_period, + pwr_status: AtomicU32::new(4), + unk_10: f32!(1.0), + actual_pstate: 1, + tgt_pstate: 1, + base_pstate_scaled: base_ps_scaled, + unk_40: 1, + max_pstate_scaled: max_ps_scaled, + min_pstate_scaled: 100, + unk_64c: 625, + pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, + pwr_filter_a: pwr_filter_a, + pwr_integral_gain: pwr.pwr_integral_gain, + pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), + max_power_1: pwr.max_power_mw.into(), + pwr_proportional_gain: pwr.pwr_proportional_gain, + pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), + pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32, + max_pstate_scaled_2: max_ps_scaled, + max_power_2: pwr.max_power_mw, + max_pstate_scaled_3: max_ps_scaled, + ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, + ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, + ppm_filter_a: ppm_filter_a, + ppm_ki_dt: pwr.ppm_ki * period_s, + unk_6fc: f32!(65536.0), + ppm_kp: pwr.ppm_kp, + pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, + max_pstate_scaled_4: max_ps_scaled, + unk_71c: f32!(0.0), + max_power_3: pwr.max_power_mw, + cur_power_mw_2: 0x0, + ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, + ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, + perf_tgt_utilization: pwr.perf_tgt_utilization, + perf_boost_min_util: pwr.perf_boost_min_util, + perf_boost_ce_step: pwr.perf_boost_ce_step, + perf_reset_iters: pwr.perf_reset_iters, + unk_774: 6, + unk_778: 1, + perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, + perf_filter_a_neg: f32!(1.0) - perf_filter_a, + perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, + perf_filter_a: perf_filter_a, + perf_filter_a2: perf_filter_a2, + perf_ki: pwr.perf_integral_gain, + perf_ki2: pwr.perf_integral_gain2, + perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), + unk_79c: f32!(95.0), + perf_kp: pwr.perf_proportional_gain, + perf_kp2: pwr.perf_proportional_gain2, + boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), + base_pstate_scaled_2: base_ps_scaled, + max_pstate_scaled_5: max_ps_scaled, + base_pstate_scaled_3: base_ps_scaled, + perf_tgt_utilization_2: pwr.perf_tgt_utilization, + base_pstate_scaled_4: base_ps_scaled, + unk_7fc: f32!(65536.0), + pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), + max_pstate_scaled_6: max_ps_scaled.into(), + max_freq_mhz: pwr.max_freq_mhz, + pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, + min_pstate_scaled_4: f32!(100.0), + max_pstate_scaled_7: max_ps_scaled, + unk_alpha_neg: f32!(0.8), + unk_alpha: f32!(0.2), + fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]), + fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, + unk_87c: cfg.da.unk_87c, + unk_880: 0x4, + unk_894: f32!(1.0), + + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_8a8: f32!(65536.0), + fast_die0_kp: pwr.fast_die0_proportional_gain, + pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, + max_pstate_scaled_8: max_ps_scaled, + max_pstate_scaled_9: max_ps_scaled, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + unk_8cc: cfg.da.unk_8cc, + max_pstate_scaled_10: max_ps_scaled, + max_pstate_scaled_11: max_ps_scaled, + unk_c2c: 1, + power_zone_count: pwr.power_zones.len() as u32, + max_power_4: pwr.max_power_mw, + max_power_5: pwr.max_power_mw, + max_power_6: pwr.max_power_mw, + avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, + avg_power_target_filter_a: avg_power_target_filter_a, + avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_clks: period_ms + * pwr.avg_power_target_filter_tc + * base_clock_khz, + avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, + avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, + avg_power_filter_a: avg_power_filter_a, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + unk_d20: f32!(65536.0), + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + max_pstate_scaled_12: max_ps_scaled, + max_pstate_scaled_13: max_ps_scaled, + max_power_7: pwr.max_power_mw.into(), + max_power_8: pwr.max_power_mw, + avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, + avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, + max_pstate_scaled_14: max_ps_scaled, + t81xx_data <- Self::t81xx_data(cfg, dyncfg), + fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]), + unk_e24: cfg.da.unk_e24, + unk_e28: 1, + fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]), + unk_163c: 1, + unk_3644: 0, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + for i in 0..tables.len { + raw.sram_k[i] = cfg.sram_k; + } + + // The only constants the G15 firmware expects in the HwDataA 0x10b4 block + // (+0x10f4 = 0x32, +0x11cc = {0, 65536.0}). + // unk_e10_0 sits at 0x10b4 (asserted in fw/initdata.rs), so +0x40 is 0x10f4 + // and +0x11c is 0x11d0. The runtime layout fills the whole block below. + { + raw.unk_e10_0.unk_40 = 0x32; + raw.unk_e10_0.unk_11c = f32!(65536.0); + } + + if let Some(csafr) = pwr.csafr.as_ref() { + for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { + raw.aux_leak_coef.cs_1[i] = *coef; + raw.aux_leak_coef.cs_2[i] = *coef; + } + + for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { + raw.aux_leak_coef.afr_1[i] = *coef; + raw.aux_leak_coef.afr_2[i] = *coef; + } + } + + for i in 0..dyncfg.id.num_clusters as usize { + if let Some(coef_a) = cfg.unk_coef_a.get(i) { + (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); + (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); + } + if let Some(coef_b) = cfg.unk_coef_b.get(i) { + (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); + (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + } + } + + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; + raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; + let filter_a = f32!(1.0) / pz.filter_tc.into(); + raw.power_zones[i].filter_a = filter_a; + raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; + raw.power_zones[i].unk_10 = 1320000000; + } + + for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { + raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; + } + + if runtime { + Self::hwdata_a_runtime(raw, pwr, max_ps_scaled, clocks_per_period_coarse); + } + + Ok(()) + })) + } + + /// Create the HwDataA structure. This mostly contains power-related configuration. + fn hwdata_a(&mut self) -> Result> { + let init = Self::hwdata_a_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataAG15V14_8_3 as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_a.len() { + matches = false; + dev_err!( + dev.as_ref(), + "!!! Hwdata A size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_a.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { + if sla[i] != dyncfg.hw_data_a[i] { + matches = false; + dev_err!(dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(dev.as_ref(), "!!! Hwdata A match"); + } + } + } + Ok(()) + }) + }) + } + + /// The runtime backend's HwDataA words that the manager layout leaves at their defaults: the + /// shader-engine controller block at +0x10b4 in its G15 order (all inputs from the device + /// tree), and the leakage coefficient and flag words of the G15 tail. Words that the + /// runtime's InitData keeps at zero are cleared. + fn hwdata_a_runtime( + raw: &mut raw::HwDataAG15V14_8_3, + pwr: &hw::PwrConfig, + max_ps_scaled: u32, + clocks_per_period_coarse: u32, + ) { + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); + let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); + let se = &mut raw.unk_e10_0; + se.unk_40 = 50; + se.unk_44 = 1; + se.gpu_se_filter_a_neg = f32!(1.0) - filter_a; + se.gpu_se_filter_1_a_neg = f32!(1.0) - filter_1_a; + se.gpu_se_filter_a = filter_a; + se.gpu_se_filter_1_a = filter_1_a; + se.gpu_se_ki_dt = pwr.se_ki * period_s; + se.gpu_se_ki_1_dt = pwr.se_ki_1 * period_s; + se.unk_68 = f32!(65536.0); + se.gpu_se_kp = pwr.se_kp; + se.gpu_se_kp_1 = pwr.se_kp_1; + se.unk_78 = 100; + se.max_pstate_scaled_1 = max_ps_scaled; + se.min_pstate_scaled = 100; + se.unk_88 = f32!(8000.0); + // The shader-engine target (ADT gpu-se-tgt, 1400 on J514S and J516S; the device tree + // may override it). + se.se_target = pwr.se_target; + se.gpu_se_filter_time_constant_ms = pwr.se_filter_time_constant * period_ms; + se.gpu_se_filter_time_constant_1_ms = pwr.se_filter_time_constant_1 * period_ms; + se.gpu_se_filter_time_constant_clks = + U64((pwr.se_filter_time_constant * clocks_per_period_coarse).into()); + se.gpu_se_filter_time_constant_1_clks = + U64((pwr.se_filter_time_constant_1 * clocks_per_period_coarse).into()); + se.unk_b0 = f32!(65536.0); + se.unk_d8 = 8000; + se.unk_dc = 4; + se.gpu_se_inactive_threshold = pwr.se_inactive_threshold; + se.gpu_se_engagement_criteria = pwr.se_engagement_criteria; + se.unk_e8 = 2; + se.unk_ec = 4; + se.gpu_se_reset_criteria = pwr.se_reset_criteria; + se.unk_11c = f32!(65536.0); + se.unk_12c = 40; + se.max_pstate_scaled_2 = max_ps_scaled; + + // No fast-die release temperature, no +0x8b8 offset and no alternate sensor mask: the + // T6030 device tree has no source for them. + raw.fast_die0_release_temp_cc = 0; + raw.unk_87c = 0; + raw.fast_die0_sensor_mask_alt = U64(0); + raw.unk_3640 = f32!(5.0).to_bits(); + // The fused core leakage coefficient of this machine (device tree), first cluster table. + if let Some(coef) = pwr.core_leak_coef.first() { + raw.cluster_tables[0][0] = coef.to_bits(); + } + raw.unk_4298 = 1; + raw.unk_429c[0] = 1; + } + + // Compare a structure against the copy m1n1 left in the reserved-memory blob, if any + // (debug aid, same as the G13/G14 builders do inline). + fn compare_blob(dev: &AsahiDevice, what: &str, raw: &T, blob: &[u8]) { + if blob.is_empty() { + return; + } + // SAFETY: `raw` is a fully initialized firmware structure; viewing it as bytes is fine. + let sla = unsafe { + core::slice::from_raw_parts(raw as *const T as *const u8, core::mem::size_of::()) + }; + if sla.len() != blob.len() { + dev_err!( + dev.as_ref(), + "!!! {} size mismatch: {} {}", + what, + sla.len(), + blob.len() + ); + } + match sla.iter().zip(blob.iter()).position(|(a, b)| a != b) { + Some(i) => dev_err!(dev.as_ref(), "!!! {} first mismatch: {i}", what), + None => dev_info!(dev.as_ref(), "!!! {} match", what), + } + } + + // Build the G15 (14.8.3) HwDataB contents without allocating the object. Board values whose + // source is not known yet are left 0 and marked TODO. + // (Plain comments: the versions macro needs `fn` right after a gating #[ver].) + fn hwdata_b_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let tables = g15.tables(pwr)?; + let runtime = g15.layout == G15Layout::Runtime; + let runtime_hwdata_b = g15.runtime_hwdata_b; + let timestamp_base = match g15.timestamp_base { + Some(base) => base, + None => mmu::kern_iova(cfg, gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + }; + Ok(try_init!(raw::HwDataBG15V14_8_3 { + // VA-layout words, +0x0/+0x8 + unk_8: U64(0x6f_00000000), + unk_18: U64(0xffc00000), + // USC start: the G15 firmware takes 0x10_00000000 at +0x10/+0x18. Like the G13 + // builder, the manager leaves 0: drm/asahi passes the USC base per queue. + unk_20: U64(0), + unk_28: U64(0), + // "Unknown page": the G15 firmware layout uses 0x2ff_ffff8000 (+0x20) in its + // 42-bit user layout; the manager keeps a 39-bit user range and maps the page + // at IOVA_UNK_PAGE, so it passes that. + unk_30: U64(mmu::IOVA_UNK_PAGE), + // FW VA of the timestamp area, +0x28 + timestamp_area_base: U64(timestamp_base), + // TODO: CSC matrices (+0x38) and border-colour pointer (+0x638) + // are left 0, as on G13 ("TODO: yuv matrices"). + // +0xa28..+0xa34: chip id + 3 words (G13 unk_454..45c) + chip_id: cfg.chip_id, + unk_454: cfg.db.unk_454, + unk_458: 0x1, + // +0xa38..+0xa64 (G13 unk_460..unk_48c). +0xa48/+0xa54/+0xa64 are 0. + // +0xa38 (GPC present, 1 on t6030) and +0xa40 (PIO supported) are 1: the + // firmware runs its power management, and so powers up the GPU cores for + // a kicked job, only if both are non-zero. +0xa50/+0xa58/+0xa60 are 1. + // Tested on J516S: an empty compute job completes with these values. + unk_460: 1, + unk_468: 1, + unk_478: 1, + unk_480: 1, + unk_488: 1, + // +0xa68: 0x5dc0 = 24000 + base_clock_khz: cfg.base_clock_hz / 1000, + // +0xa6c: 200 on slow simulation models, otherwise 1, so 1 on t6030. The + // HwDataA filter constants follow +0xa70 (the sample period in ms), which + // stays the DT value. + power_sample_period: 1, + // +0xa70: the value HwDataA +0xea8/+0xeac multiply the filter time + // constants by, i.e. the sample period in ms (inferred) + unk_a70: pwr.power_sample_period, + // TODO: +0xa74, +0xa88, +0xa90, +0xaa0, +0xac4 are board/config values whose + // source is not known yet; left 0. + unk_a78: 1, + unk_a7c: U64(1), + unk_a98: 1, + unk_a9c: 0x1f, + unk_ac8: Array::new([0, 1, 1, 0]), + // TODO: +0xaa8 and +0xb04 are condition flags; left 0. + // "UAT enabled" flag, G13 unk_554 = 1. +0xb40 + unk_554: 0x1, + // +0xb44..+0xb58 + uat_ttb_base: U64(dyncfg.uat_ttb_base), + gpu_core_id: cfg.gpu_core.ok_or(ENODEV)? as u32, + gpu_rev_id: dyncfg.id.gpu_rev_id as u32, + num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters, + max_pstate: tables.len as u32 - 1, + // +0x17e8/+0x1800/+0x1804/+0x1810; + // +0x1814 = 1 as well. + unk_17e8: 1, + timer_offset: U64(0), + unk_1800: 1, + unk_1804: 1, + unk_1810: 1, + unk_1814: 1, + // +0x17fc = 1 on t6030. + unk_17fc: 1, + // +0x1820: 48 x 0xff + all_ones_masks: Array::new([0xff; 0x30]), + // +0x1860 = 1 & ~(debug global bit 0), so 1 without debug. (inferred) + unk_1860: 1, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + // Rounded up, like the operating-point voltages. + let min_sram_mv = pwr.min_sram_microvolt.div_ceil(1000); + // Core and SRAM voltage rows of one state: every column holds the state's + // device-tree voltage (columns beyond the device tree's repeat the first one), and + // the SRAM rail never goes below the device tree's floor. + let rows = |ps: &hw::PState| -> ([u32; 8], [u32; 8]) { + let mut volts = [0u32; 8]; + for (j, v) in volts.iter_mut().enumerate() { + *v = *ps.volt_mv.get(j).unwrap_or(&ps.volt_mv[0]); + } + (volts, volts.map(|mv| mv.max(min_sram_mv))) + }; + + // p-state tables at +0xb58..+0xf9b, +0xfdc, same formulas as G13. Every + // frequency and voltage comes from the device tree. + for i in 0..tables.len { + let ps = tables.primary_state(pwr, i); + let (volts, sram) = rows(ps); + raw.frequencies[i] = ps.freq_hz / 1000000; + raw.voltages[i] = volts; + raw.voltages_sram[i] = sram; + raw.sram_k[i] = cfg.sram_k; + } + + match tables.secondary { + None => { + // One table in device-tree order: the second frequency table repeats it, + // and the index map is the identity. + for i in 0..tables.len { + raw.frequencies_2[i] = raw.frequencies[i]; + } + for i in 0..16 { + raw.unk_arr_0[i] = i as u32; + } + } + Some(secondary) => { + // The secondary table (+0xf9c), the device-tree index of every entry of + // both tables (+0x10dc, +0x111c) and the third table (+0x134c, the + // secondary table with one voltage column). + // The base state is a device-tree index, and the HwDataA base words use it as + // an index into the published table: the two must name the same state. + if pwr.perf_base_pstate as usize >= tables.len + || u32::from(tables.primary[pwr.perf_base_pstate as usize]) + != pwr.perf_base_pstate + { + return Err(EINVAL); + } + let top = tables.primary_state(pwr, tables.len - 1); + let base = tables.primary_state(pwr, pwr.perf_base_pstate as usize); + if top.pwr_mw == 0 || top.freq_hz <= base.freq_hz { + return Err(EINVAL); + } + let aux = &mut raw.aux_ps_3; + aux.max_pstate = tables.len as u32 - 1; + for i in 0..tables.len { + let ps = &pwr.perf_states[secondary[i] as usize]; + let (volts, sram) = rows(ps); + raw.frequencies_2[i] = ps.freq_hz / 1000000; + raw.unk_arr_0[i] = tables.primary[i] as u32; + raw.unk_arr_0[16 + i] = secondary[i] as u32; + aux.frequencies[i] = ps.freq_hz / 1000000; + aux.voltages[i][0] = volts[0]; + aux.voltages_sram[i][0] = sram[0]; + + // Maximum power and boost frequency of each primary entry, in percent + // of the top entry (boost relative to the base state). + let ps = tables.primary_state(pwr, i); + if i > 0 { + raw.rel_max_powers[i] = + (100 * u64::from(ps.pwr_mw) / u64::from(top.pwr_mw)) as u32; + } + if ps.freq_hz > base.freq_hz { + let f = u64::from(ps.freq_hz / 1000000 - base.freq_hz / 1000000); + let span = u64::from(top.freq_hz / 1000000 - base.freq_hz / 1000000); + raw.rel_boost_freqs[i] = (100 * f / span) as u32; + } + } + } + } + + if let Some(csafr) = pwr.csafr.as_ref() { + let aux = &mut raw.aux_ps; + aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; + aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + + for (i, ps) in csafr.perf_states_cs.iter().enumerate() { + aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.cs_voltages[i][j] = *mv; + aux.cs_voltages_sram[i][j] = sram_mv; + } + } + + for (i, ps) in csafr.perf_states_afr.iter().enumerate() { + aux.afr_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.afr_voltages[i][j] = *mv; + aux.afr_voltages_sram[i][j] = sram_mv; + } + } + } + + // +0x17c0 (u64) / +0x17c8 (u32): masks with the low N bits set + // (per-core / per-cluster enable masks, inferred). + // TODO: which counts these are; default total cores / clusters. + let low_bits = |n: u32| -> u64 { + if n >= 64 { + u64::MAX + } else { + (1u64 << n) - 1 + } + }; + raw.unit_mask_a = U64(low_bits(dyncfg.id.num_cores * dyncfg.id.num_clusters)); + raw.unit_mask_b = low_bits(dyncfg.id.num_clusters.min(32)) as u32; + + if runtime { + Self::hwdata_b_runtime(raw, runtime_hwdata_b.ok_or(EINVAL)?); + } + Ok(()) + })) + } + + // The runtime backend's HwDataB words: its GPU VA layout (USC base, unknown page), and the + // SoC's configuration words and unit masks (`words`, see `G15RuntimeHwDataB`). Those are a + // runtime InitData's values, not derived from the device tree: the T6030 ones hold for the + // boards that InitData was checked on (J514S, J516S), not necessarily for other + // configurations. + fn hwdata_b_runtime(raw: &mut raw::HwDataBG15V14_8_3, words: &G15RuntimeHwDataB) { + raw.unk_20 = U64(RUNTIME_USC_BASE); + raw.unk_28 = U64(RUNTIME_USC_BASE); + raw.unk_30 = U64(RUNTIME_UNKNOWN_PAGE); + raw.unk_454 = words.unk_454; + raw.unk_464 = words.unk_464; + raw.unk_a7c = U64(words.unk_a7c); + raw.unk_a98 = words.unk_a98; + raw.unk_abc = U64(words.unk_abc); + raw.unk_ae4 = words.unk_ae4; + raw.unk_b20 = words.unk_b20; + raw.unk_b24 = U64(words.unk_b24); + raw.unk_554 = words.unk_554; + raw.unk_17b8 = words.unk_17b8; + raw.unit_mask_a = U64(words.unit_mask_a); + raw.unit_mask_b = words.unit_mask_b; + raw.unk_1808 = words.unk_1808; + raw.unk_1818 = words.unk_1818; + } + + // Create the G15 (14.8.3) HwDataB structure. + fn hwdata_b(&mut self) -> Result> { + let init = Self::hwdata_b_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + Self::compare_blob(dev, "Hwdata B", &*raw, &dyncfg.hw_data_b); + Ok(()) + }) + }) + } + + // Build the G15 (14.8.3) Globals contents without allocating the object. Only the fields + // with a known init value are set; the rest start at 0. + fn globals_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result + 'a> { + let pwr = &dyncfg.pwr; + let tables = g15.tables(pwr)?; + let runtime = g15.layout == G15Layout::Runtime; + let reference_ppm = g15.reference_ppm; + Ok(try_init!(raw::GlobalsG15V14_8_3 { + // TODO: 0 or 7 depending on a board condition (+0x0); 0 = timer + // context switch off. + timer_cswitch: 0, + // G13 unk_24 = 0, debug = 0 (inferred) + unk_28: 0, + debug: 0, + // +0x38: constant 0x78 + unk_38: 0x78, + // G13 unk_54/56/58 moved to +0x50..+0x54 (inferred) + // TODO: these are board configuration values; G13 values for now. + unk_50: cfg.global_unk_54, + unk_52: 40, + unk_54: 0xffff, + // G13 unk_5e = 1 moved to +0x5a (a flag on G15) (inferred) + unk_5a: U32(1), + // +0x5e = max(HwDataB+0xa6c, 1) = 1. + cswitch_timer_multiplier: U32(1), + // Must be non-zero before work is submitted. + // Non-zero at boot has been accepted by the firmware on J516S. + command_submission_enabled: 1, + pending_submissions: AtomicU32::new(0), + // G13 values. TODO: the G15 defaults for +0x97c..+0x984 are not + // known. + progress_check_interval_3d: 40, + progress_check_interval_ta: 10, + progress_check_interval_cl: 250, + // G13 unk_1102c_0.._c values; same group at +0x988 (inferred) + unk_1102c_0: 1, + unk_1102c_4: 1, + unk_1102c_8: 100, + unk_1102c_c: 1, + // +0x9a0..+0x9a8 + // TODO: with the rings in Fender SRAM the GPU and Fender must stay + // powered from firmware boot on, until an SRAM backup/restore exists. + // gpu.rs update_globals() keeps idle_off_delay_ms at the same value on + // every submit. + idle_off_delay_ms: AtomicU32::new(G15_KEEP_POWERED_MS), + fender_idle_off_delay_ms: G15_KEEP_POWERED_MS, + fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, + // G13 values (+0x9b0/+0x9b4, set by setters) + cl_context_switch_timeout_ms: 40, + cl_kill_timeout_ms: 50, + // The manager leaves the FwUtil perf tunables (cfg.global_tab) and + // `soft_fault_settings` (+0xdf9) unwritten; the runtime layout sets them below. + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + if runtime { + Self::globals_runtime(raw, cfg, pwr, &tables)?; + } else if reference_ppm { + Self::globals_power_targets(raw, pwr, &tables); + } + Ok(()) + })) + } + + // The Globals power-interface targets (maximum power from the device tree, then the highest + // usable performance state scaled by 100, twice) and the performance-state cap. + fn globals_power_targets(raw: &mut raw::GlobalsG15V14_8_3, pwr: &hw::PwrConfig, tables: &PStateTables) { + let max_ps_scaled = 100 * tables.max; + raw.power_if_target = Array::new([pwr.max_power_mw, max_ps_scaled, max_ps_scaled]); + raw.perf_state_cap = max_ps_scaled; + } + + // The runtime backend's Globals: DRAM rings with finite idle-off, so the idle policy + // (imported by DeviceControl 0x13) and smart idle-off are on and the device-tree + // idle-off delays apply; the power interface targets and the performance-state cap + // (maximum power from the device tree, pstates scaled by 100); the smart idle-off + // configuration (standby timer from the device tree); and the FwUtil tunables. + fn globals_runtime( + raw: &mut raw::GlobalsG15V14_8_3, + cfg: &'static hw::HwConfig, + pwr: &hw::PwrConfig, + tables: &PStateTables, + ) -> Result { + raw.relaxed_cl_kill_timeout = 3000; + raw.debug = 1; + raw.smart_idle_off_enable = 1; + raw.unk_50 = 0xffff; + // The runtime enables command submission through its own queue protocol. + raw.command_submission_enabled = 0; + Self::globals_power_targets(raw, pwr, tables); + + let mut idle = [0u32; 10]; + idle[0] = pwr.idle_off_standby_timer; + if let Some(curve) = cfg.unk_hws2_4 { + for (slot, k) in idle[1..9].iter_mut().zip(curve.iter()) { + *slot = k.to_bits(); + } + } + idle[9] = cfg.unk_hws2_24; + raw.smart_idle_off_cfg = Array::new(idle); + raw.ut_engagement = 1; + raw.clvr_engagement = 1; + raw.keepalive_perf_threshold_rd = 100; + raw.keepalive_off_threshold_rd = 100; + + raw.idle_off_delay_ms = AtomicU32::new(pwr.idle_off_delay_ms); + raw.fender_idle_off_delay_ms = pwr.fender_idle_off_delay_ms; + raw.fw_early_wake_timeout_ms = pwr.fw_early_wake_timeout_ms; + + // CDM backoff 4, then a u32 1 and the FwUtil perf tunables from +0x9c1. + raw.cdm_backoff_timeout = 4; + raw.unk_9bd = Array::new([1, 0, 0, 0]); + if let Some(tab) = cfg.global_tab { + let mut bytes = [0u8; 0x1c]; + bytes.get_mut(..tab.len()).ok_or(EINVAL)?.copy_from_slice(tab); + raw.fwutil_default_fab_pstate = Array::new([bytes[0], bytes[1]]); + raw.fwutil_timer_period = bytes[2]; + (*raw.unk_9c4)[..0x19].copy_from_slice(&bytes[3..0x1c]); + } + raw.soft_fault_settings = U32(3); + Ok(()) + } + + // Create the G15 (14.8.3) Globals structure. + fn globals(&mut self) -> Result> { + let init = Self::globals_init(self.cfg, self.dyncfg, &self.g15)?; + let dev = self.dev; + let dyncfg = self.dyncfg; + self.alloc + .private + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { + init.chain(move |raw| { + Self::compare_blob(dev, "Globals", &*raw, &dyncfg.hw_globals); + Ok(()) + }) + }) + } + + // Create the G15 (14.8.3) RuntimePointers structure. The ring descriptors are filled in by + // the GPU manager. + fn runtime_pointers(&mut self) -> Result> { + let hwa = self.hwdata_a()?; + let hwb = self.hwdata_b()?; + let bufmgr_fw_addr = mmu::kern_iova(self.cfg, gpu::IOVA_KERN_GPU_BUFMGR_HIGH); + let rp_2d0 = self.g15.runtime_pointers_2d0(); + + let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; + buffer_mgr_ctl.vmap()?.memset(0); + + GpuObject::new_init_prealloc( + self.alloc.private.alloc_object()?, + |_ptr| { + let alloc = &mut *self.alloc; + try_init!(RuntimePointersG15V14_8_3 { + // G15 E1/E2/E3 are 0xc10/0x1248/0xe10 bytes; the G13 + // GpuGlobalStatsVtx/GpuStatsComp buffers are larger, which is harmless. + stats <- { + let alloc = &mut *alloc; + try_init!(StatsG15V14_8_3 { + vtx: alloc.private.new_default::()?, + frag: alloc.private.new_init( + pin_init::init_zeroed::(), + |_inner, _ptr| { + try_init!(raw::GpuGlobalStatsFragG15V14_8_3 { + total_cmds: 0, + unk_4: 0, + stats: Default::default(), + }) + } + )?, + comp: alloc.private.new_default::()?, + }) + }, + + hwdata_a: hwa, + hwdata_b: hwb, + + // BRN (erratum) table. + // TODO: size and contents unknown; one zeroed page (no errata). + brn_table: alloc.private.array_empty_tagged(0x4000, b"IBRN")?, + // E7: 0x88 bytes, zeroed + unkptr_e7: alloc.private.array_empty_tagged(0x88, b"IE7_")?, + // E5: 0x60 bytes, zeroed + unkptr_e5: alloc.private.array_empty_tagged(0x60, b"IE5_")?, + // UMA page-pool descriptor table (RuntimePointers +0x2c0/+0x2c8). + // TODO: size/format unknown; one zeroed page. The firmware takes a + // GPU VA and a FW VA; both point at this kernel mapping for now. + uma_pool_desc: alloc.shared.array_empty_tagged(0x4000, b"IUMA")?, + + buffer_mgr_ctl, + buffer_mgr_ctl_low_mapping: None, + buffer_mgr_ctl_high_mapping: None, + }) + }, + |inner, _ptr| { + try_init!(raw::RuntimePointersG15V14_8_3 { + hwdata_b: inner.hwdata_b.gpu_pointer(), + brn_table: inner.brn_table.gpu_pointer(), + unkptr_e7: inner.unkptr_e7.gpu_pointer(), + + pipes: Default::default(), + device_control: Default::default(), + event: Default::default(), + fw_log: Default::default(), + ktrace: Default::default(), + stats: Default::default(), + fwlog_buf: None, + __pad_200: Default::default(), + // FWLog enable: the firmware writes no log records while this is 0. + // Opt-in via asahi.g15_debug bit 43. + unk_230: crate::m3_params::g15_debug(crate::m3_params::G15Debug::FwLog) as u32, + + stats_vtx: inner.stats.vtx.gpu_pointer(), + stats_frag: inner.stats.frag.gpu_pointer(), + stats_comp: inner.stats.comp.gpu_pointer(), + unkptr_e5: inner.unkptr_e5.gpu_pointer(), + __pad_254: Default::default(), + // Optional pointer, left 0 + unkptr_2a8: U64(0), + buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), + buffer_mgr_ctl_fw_addr: U64(bufmgr_fw_addr), + uma_pool_desc_gpu_addr: U64(inner.uma_pool_desc.gpu_va().get()), + uma_pool_desc_fw_addr: U64(inner.uma_pool_desc.gpu_va().get()), + // TODO: board configuration words; +0x2d4 left 0. + unk_2d0: rp_2d0, + unk_2d4: 0, + __pad_2d8: Default::default(), + // +0x3b0: 0xff, then 0x90 zero bytes + cswitch_hist_ctl: 0xff, + unk_3b1: Default::default(), + hwdata_a: inner.hwdata_a.gpu_pointer(), + // TODO: 16 board configuration bytes; left 0. + unk_449: Default::default(), + __pad_459: Default::default(), + unk_46d: Default::default(), + __pad_48d: Default::default(), + }) + }, + ) + } + + // Fill the G15 P0 status block. Host-initialised words are 0 except +0x45c4, which arms the + // firmware work-scheduler callbacks, and in the runtime layout the noise-suppression idle + // power-off state (+0x4), which the runtime's InitData starts at 1. + fn status_block_fill(raw: &mut raw::StatusBlock, g15: &G15Options) { + raw.flags.sched_callbacks = 1; + if g15.layout == G15Layout::Runtime { + raw.noise_suppression_idle_pwroff = 1; + } + } + + // Create the G15 P0 status block. + fn status_block(&mut self) -> Result> { + let g15 = self.g15; + let mut block = self.alloc.shared.new_default::()?; + block.with_mut(|raw, _inner| Self::status_block_fill(raw, &g15)); + Ok(block) + } + + // Build the G15 P1 power-controller block without allocating it. The fields keep the order + // and the formulas of the G13 13.5 Globals power fields (inferred). + fn power_ctl_block_init( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> impl Init + 'a { + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let runtime = g15.layout == G15Layout::Runtime; + let reference_ppm = runtime || g15.reference_ppm; + + try_init!(raw::PowerCtlBlock { + power_zone_count: pwr.power_zones.len() as u32, + avg_power_filter_tc_periods: pwr.avg_power_filter_tc_ms / period_ms, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, + // G13 Globals unk_89bc..unk_89e0 order (inferred) + fast_die_temp_target: cfg.da.unk_8cc, + fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, + unk_b0: cfg.da.unk_87c, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + fast_die0_kp: pwr.fast_die0_proportional_gain, + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + // +0xc0/+0xc4 are board values at init (also mirrored at HwDataA + // +0x934/+0x93c, G13 unk_8e8/unk_8f0, which the G13 path leaves 0). + // TODO: real values unknown; default 0 as in the G13 HwDataA mirror. + unk_c0: 0, + unk_c4: 0, + // +0xc8..+0xd8, manager layout: the G13 unk_89e0 = 1 flag (on G15 a + // condition flag), then the PPM target and the PPM kp. The PPM controller + // normally receives them at runtime; the G13 Globals values keep it from + // starting with a zero target. + ppm_words: Array::new([1, pwr.max_power_mw, pwr.ppm_kp.to_bits(), 0, 0]), + // +0xec: 1 at init + se_engagement: 1, + ..Zeroable::init_zeroed() + }) + .chain(move |raw| { + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc = pz.filter_tc; + } + if runtime { + // Runtime layout: no fast-die release temperature or +0xb0 offset (the + // T6030 device tree has no source for them). + raw.fast_die0_release_temp = 0; + raw.unk_b0 = 0; + } + if reference_ppm { + // 0 at +0xc8, then the enable flag, the PPM target and the PPM kp and ki*dt. + let ppm_ki_dt = pwr.ppm_ki * period_s; + raw.ppm_words = Array::new([ + 0, + 1, + pwr.max_power_mw, + pwr.ppm_kp.to_bits(), + ppm_ki_dt.to_bits(), + ]); + } + Ok(()) + }) + } + + // Create the G15 P1 power-controller block. + fn power_ctl_block(&mut self) -> Result> { + let init = Self::power_ctl_block_init(self.cfg, self.dyncfg, &self.g15); + self.alloc + .shared + .new_init(pin_init::init_zeroed(), move |_inner, _ptr| init) + } + + // Create the G15 P2 debug block. ktrace stays disabled, as on G13. + fn debug_block(&mut self) -> Result> { + self.alloc.shared.new_default::() + } + + /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. + fn uat_level_info( + cfg: &'static hw::HwConfig, + index_shift: usize, + num_entries: usize, + ) -> raw::UatLevelInfo { + raw::UatLevelInfo { + index_shift: index_shift as _, + unk_1: 14, + unk_2: 14, + unk_3: 8, + unk_4: 0x4000, + num_entries: num_entries as _, + unk_8: U64(1), + unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), + index_mask: U64(((num_entries - 1) << index_shift) as u64), + } + } + + /// The three UatLevelInfo records of InitData: L1 ("page catalogue") entries are 8 with a + /// 39-bit IAS and 64 with the 42-bit G15 IAS (index mask 0x3f0_0000_0000). + pub(crate) fn uat_levels(cfg: &'static hw::HwConfig) -> [raw::UatLevelInfo; 3] { + let uat_l1_entries = 1usize << (cfg.uat_ias as usize - 36); + [ + Self::uat_level_info(cfg, 36, uat_l1_entries), + Self::uat_level_info(cfg, 25, 2048), + Self::uat_level_info(cfg, 14, 2048), + ] + } + + /// Build the top-level InitData object. + #[inline(never)] + pub(crate) fn build(&mut self) -> Result>> { + { + let pwr = &self.dyncfg.pwr; + let tables = self.g15.tables(pwr)?; + dev_info!( + self.dev.as_ref(), + "G15: performance cap {} -> {} state {} of 1..={} ({} MHz)\n", + self.g15.cap.unwrap_or(0), + if tables.secondary.is_some() { "voltage-sorted" } else { "device-tree" }, + tables.max, + tables.len - 1, + tables.max_frequency_khz(pwr) / 1000 + ); + } + let runtime_pointers = self.runtime_pointers()?; + let globals = self.globals()?; + // 14.x: FwStatus lives inside the P0 status block + let fw_status = self.status_block()?; + let power_ctl_block = self.power_ctl_block()?; + let debug_block = self.debug_block()?; + let shared_ro = &mut self.alloc.shared_ro; + + let obj = self.alloc.private.new_init( + try_init!(InitDataG15V14_8_3 { + unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, + runtime_pointers, + globals, + fw_status, + power_ctl_block, + debug_block, + }), + |inner, _ptr| { + let cfg = &self.cfg; + try_init!(raw::InitDataG15V14_8_3 { + ver_info: Array::new([0x0490, 0x8380, 0xe21e, 0x0c08]), + unk_buf: inner.unk_buf.gpu_pointer(), + unk_8: 0, + unk_c: 0, + runtime_pointers: inner.runtime_pointers.gpu_pointer(), + globals: inner.globals.gpu_pointer(), + // InitData +0x28/+0x2c: u32 0, u32 1 + unk_28: 0, + unk_2c: 1, + uat_page_size: 0x4000, + uat_page_bits: 14, + uat_num_levels: 3, + uat_level_info: Array::new(Self::uat_levels(cfg)), + // TODO: pmap descriptor +0x7c, meaning unknown; left 0. + unk_94: 0, + __pad_98: Default::default(), + debug_block: inner.debug_block.gpu_pointer(), + status_block: inner.fw_status.gpu_pointer(), + power_ctl_block: inner.power_ctl_block.gpu_pointer(), + }) + }, + )?; + Ok(KBox::new(obj, GFP_KERNEL)?) + } +} + +/// G15 InitData contents built into plain memory, for a backend that places the firmware objects +/// itself (the M3 runtime backend, whose object layout is fixed). +pub(crate) struct G15Contents { + pub(crate) hwdata_a: KVBox, + pub(crate) hwdata_b: KVBox, + pub(crate) globals: KVBox, + pub(crate) power_ctl_block: KVBox, + pub(crate) status_block: KVBox, + pub(crate) uat_levels: [raw::UatLevelInfo; 3], + /// The published performance-state tables. + pub(crate) tables: PStateTables, +} + +impl<'a> InitDataBuilderG15V14_8_3<'a> { + /// Build the G15 InitData contents with the same builders the manager backend uses, into + /// plain memory. Pointer fields stay zero; the caller relocates them. + pub(crate) fn g15_contents( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + g15: &G15Options, + ) -> Result { + let mut status_block = KVBox::init(pin_init::init_zeroed(), GFP_KERNEL)?; + Self::status_block_fill(&mut status_block, g15); + Ok(G15Contents { + hwdata_a: KVBox::init(Self::hwdata_a_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + hwdata_b: KVBox::init(Self::hwdata_b_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + globals: KVBox::init(Self::globals_init(cfg, dyncfg, g15)?, GFP_KERNEL)?, + power_ctl_block: KVBox::init(Self::power_ctl_block_init(cfg, dyncfg, g15), GFP_KERNEL)?, + status_block, + uat_levels: Self::uat_levels(cfg), + tables: g15.tables(&dyncfg.pwr)?, + }) + } +} + +/// The bytes of a firmware structure built by this module. +/// +/// The structures are plain `repr(C)` data, and every builder starts from a zeroed object, so +/// no byte is uninitialized. +pub(crate) fn raw_bytes(value: &T) -> &[u8] { + // SAFETY: `value` is a live, initialized `T` (see above); the slice covers exactly it and + // borrows it for its lifetime. + unsafe { core::slice::from_raw_parts((value as *const T).cast::(), size_of::()) } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs new file mode 100644 index 00000000000000..2fa384c910688e --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -0,0 +1,1491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +use core::mem::{offset_of, size_of}; + +use kernel::{ + c_str, device, + device::Core, + platform, + prelude::*, + time::{delay::fsleep, Delta}, +}; + +use crate::{ + float::F32, + fw::initdata::raw, + hw, initdata, + initdata::{G15Layout, G15Options}, + m3_config::Records, + m3_firmware::Firmware, + m3_init_layout as init, + m3_init_storage as storage, + m3_memory::Buffer, + m3_params::{self, G15Debug, InitDataParam, ThermalMode}, + m3_soc::{IoMapping, PstateTable, Soc}, + t8122_hwdata, + t8122_start::Experiment, +}; + +// The T8122 HwDataA +0x418c block (`t8122_hwdata`) covers exactly the G13-era leakage +// coefficient fields from core_leak_coef[4] to the end of aux_leak_coef, which the G15 builder +// leaves zero, and nothing the builder fills. +const _: () = { + type A = raw::HwDataAG15V14_8_3; + assert!(offset_of!(A, core_leak_coef) == 0x417c); + assert!(offset_of!(A, sram_leak_coef) == 0x419c); + assert!(offset_of!(A, aux_leak_coef) == 0x41bc); + assert!(t8122_hwdata::OFFSET == offset_of!(A, core_leak_coef) + 4 * size_of::()); + assert!( + t8122_hwdata::OFFSET + t8122_hwdata::SIZE + == offset_of!(A, aux_leak_coef) + size_of::() + ); + assert!(t8122_hwdata::OFFSET + t8122_hwdata::SIZE <= size_of::()); +}; + +/// Owner indices (`m3_init_storage`) of the objects the typed builders generate. +pub(crate) const INITDATA: usize = storage::ROOT; +pub(crate) const RUNTIME_POINTERS: usize = storage::RUNTIME_POINTERS; +pub(crate) const HWDATA: usize = storage::HARDWARE_DATA; +pub(crate) const GLOBALS: usize = storage::GLOBALS; +pub(crate) const GLOBALS_POWER: usize = storage::GLOBALS_POWER; +pub(crate) const STATUS: usize = storage::CONTROL_REGION; +pub(crate) const DEBUG: usize = storage::RUNTIME_FLAGS; +/// HwDataA follows HwDataB inside the HwData object. +pub(crate) const HWDATA_A: usize = 0x4580; + +/// Offset of HwDataB's IO-mapping table and the size of one entry. +const IO_MAPPINGS: usize = offset_of!(raw::HwDataBG15V14_8_3, io_mappings); +const IO_MAPPING_SIZE: usize = size_of::(); +/// Number of IO-mapping slots of HwDataB. +const IO_MAPPING_SLOTS: usize = 31; + +/// The T6030 firmware IO mappings of the runtime backend (HwDataB slot, physical address, total +/// size, element size, writable). The runtime maps IO with its own IO-mapping table +/// (`m3_init_storage::T6030_IOMAPS`), which must cover every entry (checked when the contents +/// are built). The physical addresses are T6030 SoC register blocks, the same ones the G15 manager +/// maps (`hw::t6030`). +pub(crate) const T6030_IO_MAPPINGS: [IoMapping; 15] = [ + (0, 0x2_90d0_0000, 0x14_4000, 0x14_4000, true), // Fender + (1, 0x2_0e10_1000, 1, 1, false), // AIC timer + (2, 0x3_5101_4000, 0x4000, 0x4000, true), // AIC software interrupts + (3, 0x2_9000_0000, 0x2_0000, 0x2_0000, true), // RGX + (7, 0x3_502b_c000, 0x1000, 0x1000, false), // analog temperature sensors + (9, 0x2_90e0_8000, 0x8000, 0x8000, true), // metrology sensors + (11, 0x2_2000_0000, 0x12_a000, 0x9_5000, true), // memory cache, two instances + (12, 0x3_5104_c000, 0x4000, 0x4000, false), // AIC banked registers + (18, 0x3_503d_0000, 0x4000, 0x4000, true), // telemetry dashboard + (19, 0x3_503c_0000, 0x4000, 0x4000, false), // telemetry dashboard (read) + (20, 0x3_503d_8000, 0x4000, 0x4000, true), // telemetry dashboard (config) + (23, 0x2_9300_0000, 0x40_0000, 0x40_0000, true), // AFR registers + (25, 0x3_0945_c000, 1, 1, true), // ANE doorbell + (26, 0x3_5028_0000, 0x8000, 0x8000, false), // PMS metrology sensors + (29, 0x2_90e5_c000, 0x4000, 0x4000, false), // GFX clock generator +]; + +/// The T8122 firmware IO mappings of the runtime backend, in the same form: the blocks of +/// `hw::t8122`, with the exact register, total size and element size the G15G firmware takes in +/// each slot. The AIC timer entry is the T6030 one, the same register on every SoC. +/// `fill_io_mappings` checks every entry against the runtime's IO maps of the SoC +/// ([`T8122_IOMAPS`]) before writing it. +pub(crate) const T8122_IO_MAPPINGS: [IoMapping; 12] = [ + (0, 0x2_90d0_0000, 0x10_4000, 0x10_4000, true), // Fender + (1, 0x2_0e10_1000, 1, 1, false), // AIC timer + (2, 0x2_d101_4048, 1, 1, true), // AIC software interrupt register + (3, 0x2_9000_0000, 0x2_0000, 0x2_0000, true), // RGX + (9, 0x2_90e0_8000, 0x8000, 0x8000, true), // metrology sensors + (10, 0x2_90d0_d000, 0x1000, 0x1000, true), // GM GIFAF registers + (11, 0x2_2000_0000, 0xa_a000, 0x5_5000, true), // memory cache, two instances + (18, 0x2_d03d_0000, 0x1000, 0x1000, true), // telemetry dashboard + (19, 0x2_d03c_0000, 0x2000, 0x2000, false), // telemetry dashboard (read) + (25, 0x3_1145_c000, 1, 1, true), // ANE doorbell + (26, 0x2_d028_0000, 0x8000, 0x8000, false), // PMS metrology sensors + (29, 0x2_90e1_c000, 0x4000, 0x4000, false), // GPU clock generator +]; + +/// The T8122 runtime's IO maps (`m3_soc::T8122.iomaps`): its IO mappings, laid out as the T6030 +/// ones are. +pub(crate) static T8122_IOMAPS: [storage::IoMap; 12] = + storage::pack_iomaps(&T8122_IO_MAPPINGS, storage::IOMAP_BASE); + +/// Whether every entry of `mappings` is a whole number of elements in the HwConfig block of the +/// same slot, with the same writability: the check `fill_io_mappings` makes when it builds the +/// InitData. +const fn same_blocks(mappings: &[IoMapping], cfg: &hw::HwConfig) -> bool { + let mut i = 0; + while i < mappings.len() { + let (slot, phys, total, element, writable) = mappings[i]; + if slot >= cfg.io_mappings.len() || element == 0 || total % element != 0 { + return false; + } + match cfg.io_mappings[slot] { + Some(block) + if block.base as u64 & !0x3fff == phys & !0x3fff && block.writable == writable => {} + _ => return false, + } + i += 1; + } + true +} + +const _: () = assert!(same_blocks(&T8122_IO_MAPPINGS, &hw::t8122::HWCONFIG_T8122)); + +// The T6030 IO maps are the packed IO maps of its IO mappings, the layout the T8122 start +// experiment derives its own from (`storage::pack_iomaps`). +const _: () = assert!(storage::same_iomaps( + &storage::pack_iomaps(&T6030_IO_MAPPINGS, storage::IOMAP_BASE), + &storage::T6030_IOMAPS +)); + +/// Highest performance state device-tree InitData may use without the driver's thermal limit +/// (`asahi.m3_thermal=off`), and the runtime cap the thermal limit starts at and falls back to +/// (the "safe cap"). The InitData's fast-die temperature controller words stay 0, so the +/// firmware does not limit the die temperature by itself. +pub(crate) const ADT_MAX_PSTATE_LIMIT: u32 = 5; + +/// The performance states the runtime backend lets the firmware use. +#[derive(Copy, Clone, Debug)] +pub(crate) struct PstatePolicy { + /// Highest entry of the published table. + pub(crate) table_max: u32, + /// Frequency of the highest entry of the published table, in MHz. + pub(crate) table_max_mhz: u32, + /// The ceiling: the highest entry the firmware is given (the InitData's highest-state + /// words). A reported state above it marks the GPU failed. + pub(crate) max: u32, + /// The runtime cap at boot (the Globals cap words): the highest entry the firmware may use + /// until the thermal limit changes it. Equal to `max` unless the thermal limit is on or + /// holding, which start here and never go below it. + pub(crate) safe: u32, + /// The entry requested at boot, at most `safe`. + pub(crate) boot: u32, + /// Frequency of the ceiling entry, in MHz. + pub(crate) max_mhz: u32, + /// Frequency of the `safe` entry, in MHz. + pub(crate) safe_mhz: u32, + /// Frequency of the boot entry, in MHz. + pub(crate) boot_mhz: u32, + /// Frequency of every entry of the published table, in MHz (entry 0 is the off state). + pub(crate) freqs: [u32; 16], + /// `asahi.m3_thermal`. + pub(crate) thermal: ThermalMode, +} + +impl PstatePolicy { + /// The highest frequency the GPU may run at this boot, for userspace: the ceiling's when + /// the thermal limit is on, the runtime cap's otherwise. + pub(crate) fn reported_max_mhz(&self) -> u32 { + match self.thermal { + ThermalMode::On => self.max_mhz, + ThermalMode::Off | ThermalMode::Hold => self.safe_mhz, + } + } + + /// Clamp the `asahi.m3_max_pstate` and `asahi.m3_boot_pstate` settings to the published table + /// of `hwdata` (the generated HwData object image), and to `ceiling` when given (the T8122 + /// start experiment's `asahi.t8122_pstate_cap`). + fn new(dev: &device::Device, hwdata: &[u8], ceiling: Option) -> Result { + let hwb_max = offset_of!(raw::HwDataBG15V14_8_3, max_pstate); + let hwb_freq = offset_of!(raw::HwDataBG15V14_8_3, frequencies); + let table_max = read_u32(hwdata, hwb_max)?; + if table_max == 0 || table_max >= 16 { + return Err(EINVAL); + } + let freq = |i: u32| read_u32(hwdata, hwb_freq + 4 * i as usize); + // Unset: the thermal limit is on. + let thermal = m3_params::thermal_param().unwrap_or(ThermalMode::On); + // Unset: the whole table when the thermal limit bounds the runtime cap, the default cap + // otherwise. + let requested = m3_params::max_pstate_param().unwrap_or(match thermal { + ThermalMode::Hold | ThermalMode::On => u64::from(table_max), + ThermalMode::Off => m3_params::MAX_PSTATE_DEFAULT, + }); + let mut max = requested.clamp(1, u64::from(table_max)) as u32; + if u64::from(max) != requested { + dev_info!( + dev, + "M3: asahi.m3_max_pstate={} clamped to {} (the table has states 1..={})\n", + requested, + max, + table_max + ); + } + if thermal == ThermalMode::Off && max > ADT_MAX_PSTATE_LIMIT { + dev_info!( + dev, + "M3: performance cap {} lowered to {}: device-tree InitData leaves the fast-die temperature controller off and the thermal limit is off (asahi.m3_thermal=off)\n", + max, + ADT_MAX_PSTATE_LIMIT + ); + max = ADT_MAX_PSTATE_LIMIT; + } + if let Some(ceiling) = ceiling { + if max > ceiling { + dev_info!( + dev, + "M3 G15G start: performance cap {} lowered to {} (asahi.t8122_pstate_cap)\n", + max, + ceiling + ); + max = ceiling; + } + } + // Without the thermal limit the runtime cap is the ceiling; with it, the runtime cap + // starts at (and never goes below) the cap used without it. + let safe = match thermal { + ThermalMode::Off => max, + ThermalMode::Hold | ThermalMode::On => max.min(ADT_MAX_PSTATE_LIMIT), + }; + let boot = m3_params::boot_pstate_param() + .unwrap_or(u64::from(safe)) + .clamp(1, u64::from(safe)) as u32; + let mut freqs = [0u32; 16]; + for (i, f) in freqs.iter_mut().enumerate().take(table_max as usize + 1) { + *f = freq(i as u32)?; + } + Ok(PstatePolicy { + table_max, + table_max_mhz: freqs[table_max as usize], + max, + safe, + boot, + max_mhz: freqs[max as usize], + safe_mhz: freqs[safe as usize], + boot_mhz: freqs[boot as usize], + freqs, + thermal, + }) + } +} + +/// The runtime backend's InitData contents: the generated object images and the +/// performance-state policy. +pub(crate) struct Contents { + /// Generated images, per owner; None for owners that stay zero. Only the three image + /// owners' images are uploaded; the records' are checked. + images: KVec>>, + /// The performance states the firmware may use. + pub(crate) pstates: PstatePolicy, + /// The runtime's IO maps of the SoC (`Soc::iomaps`), which cover the InitData IO mappings. + pub(crate) iomaps: &'static [storage::IoMap], + /// The accepted performance-state table (`Soc::pstates`): states above the off state, and + /// the frequency of the highest one in MHz. + pub(crate) table: (u32, u32), + /// The HwDataB slots whose IO maps the firmware gets read-only, as a bit mask. 0 (every IO + /// map read-write) except in the T8122 start experiment. + pub(crate) read_only_slots: u32, + /// The HwData object's placement and mapping (`Soc::hwdata_object`). + pub(crate) hwdata_object: Option, +} + +fn read_u32(bytes: &[u8], offset: usize) -> Result { + let word = bytes.get(offset..offset + 4).ok_or(EINVAL)?; + Ok(u32::from_le_bytes(word.try_into().map_err(|_| EINVAL)?)) +} + +fn write(bytes: &mut [u8], offset: usize, data: &[u8]) -> Result { + bytes + .get_mut(offset..offset + data.len()) + .ok_or(EINVAL)? + .copy_from_slice(data); + Ok(()) +} + +impl Contents { + /// Build the InitData contents from this board's device tree (`asahi.m3_initdata`), before + /// any GPU register is touched. + /// + /// `experiment` is the T8122 start experiment's values (`t8122_start`), on an armed T8122 + /// only; it then stands in for the SoC table's missing ones. `t6031` is the same for an + /// armed T6031 (`t6031_start`). + pub(crate) fn select( + pdev: &platform::Device, + firmware: &Firmware, + soc: &Soc, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + ) -> Result { + let dev = pdev.as_ref(); + let param = m3_params::initdata_param(); + check_version(dev, firmware, experiment, t6031)?; + let iomaps = match experiment { + Some(e) => e.iomaps(), + None => match t6031 { + Some(e) => e.iomaps(), + None => soc.iomaps.ok_or(ENODEV)?, + }, + }; + storage::validate_iomaps(iomaps).map_err(|_| EINVAL)?; + let images = build_images(dev, firmware, soc, experiment, t6031, iomaps)?; + let hwdata = images.get(HWDATA).and_then(|i| i.as_deref()).ok_or(EINVAL)?; + let pstates = PstatePolicy::new( + dev, + hwdata, + experiment.map(|e| e.pstate_cap()).or_else(|| t6031.map(|e| e.pstate_cap())), + )?; + let table = match soc.pstates { + PstateTable::Fixed { states, top_mhz } => (states, top_mhz), + PstateTable::DeviceTree => crate::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { + dev_err!( + dev, + "M3: the device tree's operating points give no performance-state table\n" + ); + EINVAL + })?, + }; + if (pstates.table_max, pstates.table_max_mhz) != table { + dev_err!( + dev, + "M3: performance-state table 1..={} up to {} MHz, the {} table is 1..={} up to {} MHz\n", + pstates.table_max, + pstates.table_max_mhz, + soc.name, + table.0, + table.1 + ); + return Err(EINVAL); + } + // The published states never go above the device tree's highest operating point. + if let Some(dt_khz) = crate::m3_board::max_frequency_khz(pdev) { + if 1000 * pstates.max_mhz > dt_khz { + dev_err!( + dev, + "M3: performance cap {} MHz is above the device tree's highest operating point ({} kHz)\n", + pstates.max_mhz, + dt_khz + ); + return Err(EINVAL); + } + } + dev_info!( + dev, + "M3: InitData contents: generated from the device tree (asahi.m3_initdata={}); performance states 1..={}, cap {} ({} MHz), boot request {} ({} MHz); thermal limit {}, firmware highest state {} ({} MHz)\n", + match param { + InitDataParam::Auto => "auto", + InitDataParam::Adt => "adt", + }, + pstates.table_max, + pstates.safe, + pstates.safe_mhz, + pstates.boot, + pstates.boot_mhz, + match pstates.thermal { + ThermalMode::Off => "off", + ThermalMode::Hold => "holding", + ThermalMode::On => "on", + }, + pstates.max, + pstates.max_mhz + ); + let read_only_slots = experiment + .map(|e| e.read_only_slots()) + .or_else(|| t6031.map(|e| e.read_only_slots())) + .unwrap_or(0); + let hwdata_object = match t6031 { + Some(e) => e.hwdata_object(), + None => soc.hwdata_object, + }; + Ok(Contents { images, pstates, iomaps, table, read_only_slots, hwdata_object }) + } + + /// Check the contents before they are uploaded. + pub(crate) fn check_for_upload(&self) -> Result { + if self.images.len() == storage::COUNT { + Ok(()) + } else { + Err(EINVAL) + } + } + + /// The initial contents of owner `index`, allocated as `a`: its generated image, or nothing + /// (zeroed) for owners without one. + pub(crate) fn initial<'a>(&'a self, index: usize, a: &storage::Allocation) -> Result<&'a [u8]> { + if a.initial == storage::Initial::Zero { + return Ok(&[]); + } + let image = self.images.get(index).and_then(|i| i.as_deref()).ok_or(EINVAL)?; + if image.len() != a.size { + return Err(EINVAL); + } + Ok(image) + } +} + +/// The fixed allocation of owner `index`, as a firmware region moved by `delta`. +fn allocation_region(index: usize, delta: u64) -> Result { + let a = storage::allocation(index).map_err(|_| EINVAL)?; + init::Region::new(a.address.checked_add(delta).ok_or(EINVAL)?, a.size).map_err(|_| EINVAL) +} + +/// The records the runtime constructs, encoded for its fixed allocation table moved by `delta`. +pub(crate) fn allocation_records(delta: u64) -> Result { + Records::encode(|index| allocation_region(index, delta)) +} + +/// The version word of the constructed InitData root. +pub(crate) fn constructed_version() -> Result { + let records = allocation_records(0)?; + Ok(u64::from_le_bytes(records.root[..8].try_into().map_err(|_| EINVAL)?)) +} + +/// Refuse to go on when the InitData root the runtime constructs carries another version than +/// the loaded firmware expects. The T8122 start experiment gives the firmware the version of +/// `asahi.t8122_initdata_version` instead (`m3_config::Config::new` writes it into the root). +fn check_version( + dev: &device::Device, + firmware: &Firmware, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, +) -> Result { + let version = constructed_version()?; + if version != firmware.initdata_magic && experiment.is_some() { + dev_warn!( + dev, + "M3 G15G start: InitData version {:#x} (asahi.t8122_initdata_version) replaces the constructed {:#x}\n", + firmware.initdata_magic, + version + ); + return Ok(()); + } + if version != firmware.initdata_magic && t6031.is_some() { + dev_warn!( + dev, + "M3 G15C start: InitData version {:#x} replaces the constructed {:#x}\n", + firmware.initdata_magic, + version + ); + return Ok(()); + } + if version != firmware.initdata_magic { + dev_err!( + dev, + "M3: the constructed InitData version {:#x} is not the one the loaded firmware expects ({:#x})\n", + version, + firmware.initdata_magic + ); + return Err(ENODEV); + } + Ok(()) +} + +/// Whether the runtime probe stops before the GPU coprocessor starts (`asahi.g15_debug` bit +/// 53). Logs the stop. +pub(crate) fn stop_before_asc(dev: &device::Device) -> bool { + let stop = m3_params::g15_debug(G15Debug::StopBeforeAsc); + if stop { + dev_info!( + dev, + "M3: asahi.g15_debug bit 53: stopping before the GPU coprocessor starts; the firmware was not started\n" + ); + } + stop +} + +/// The GPU identity the runtime backend admits for `soc` (its SGX ID words, checked in +/// `m3_device` before the firmware starts): on T6030, G15, variant S, revision B1, one die with +/// two clusters of ten core slots. +fn soc_identity( + soc: &Soc, + cfg: &'static hw::HwConfig, + rev_id: Option, +) -> hw::GpuIdConfig { + hw::GpuIdConfig { + gpu_gen: hw::GpuGen::G15, + gpu_variant: soc.gpu_variant, + usc_generation: 3, + gpu_hal_generation: hw::GpuHalGeneration::Legacy, + gpu_rev: soc.gpu_revision, + gpu_rev_id: rev_id.unwrap_or(soc.gpu_revision_id), + num_dies: cfg.num_dies, + num_clusters: cfg.max_num_clusters, + num_cores: cfg.max_num_cores, + num_frags: cfg.max_num_frags, + num_gps: cfg.max_num_gps, + total_active_cores: 0, + core_masks: KVec::new(), + core_masks_packed: KVec::new(), + } +} + +pub(crate) const BOARD_DATA: [usize; 3] = [HWDATA, GLOBALS, GLOBALS_POWER]; + +/// The pointers of the constructed records that the typed firmware structures place: record +/// owner, offset, target owner, offset into the target. +const RECORD_POINTERS: [(usize, usize, usize, usize); 7] = { + type InitData = raw::InitDataG15V14_8_3<'static>; + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + [ + (INITDATA, offset_of!(InitData, runtime_pointers), RUNTIME_POINTERS, 0), + (INITDATA, offset_of!(InitData, globals), GLOBALS, 0), + (INITDATA, offset_of!(InitData, debug_block), DEBUG, 0), + (INITDATA, offset_of!(InitData, status_block), STATUS, 0), + (INITDATA, offset_of!(InitData, power_ctl_block), GLOBALS_POWER, 0), + (RUNTIME_POINTERS, offset_of!(RuntimePointers, hwdata_b), HWDATA, 0), + (RUNTIME_POINTERS, offset_of!(RuntimePointers, hwdata_a), HWDATA, HWDATA_A), + ] +}; + +/// The bytes of a constructed record, by owner. +pub(crate) fn record_bytes(records: &Records, owner: usize) -> Option<&[u8]> { + match owner { + INITDATA => Some(&records.root[..]), + RUNTIME_POINTERS => Some(&records.runtime[..]), + STATUS => Some(&records.control[..]), + _ => None, + } +} + +fn check_layout() -> Result { + type InitData = raw::InitDataG15V14_8_3<'static>; + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + let records = allocation_records(0)?; + let pointer = |owner: usize, offset: usize| -> Option { + let bytes = record_bytes(&records, owner)?.get(offset..offset + 8)?; + Some(u64::from_le_bytes(bytes.try_into().ok()?)) + }; + let mut ok = true; + for (owner, offset, target, at) in RECORD_POINTERS { + let a = storage::allocation(target).map_err(|_| EINVAL)?; + ok &= at < a.size && pointer(owner, offset) == Some(a.address + at as u64); + } + for index in 0..storage::COUNT { + let a = storage::allocation(index).map_err(|_| EINVAL)?; + let board = BOARD_DATA.contains(&index); + ok &= board == (a.initial != storage::Initial::Zero); + } + let size = |index: usize| storage::allocation(index).map_or(0, |a| a.size); + let ok = ok + && size(INITDATA) >= size_of::() + && size(RUNTIME_POINTERS) >= offset_of!(RuntimePointers, hwdata_a) + 8 + && HWDATA_A >= size_of::() + && size(HWDATA) >= HWDATA_A + size_of::() + && size(GLOBALS) >= size_of::() + && size(GLOBALS_POWER) >= size_of::() + && size(STATUS) >= size_of::() + && size(DEBUG) >= size_of::(); + if ok { Ok(()) } else { Err(EINVAL) } +} + +/// Fill HwDataB's IO-mapping table (the runtime fills the virtual addresses when it maps them), +/// checking every entry against the runtime's IO mapping of the same slot (`iomaps`). +fn fill_io_mappings( + dev: &device::Device, + cfg: &'static hw::HwConfig, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + mappings: &[IoMapping], + iomaps: &[storage::IoMap], + hwdata: &mut [u8], +) -> Result { + for (i, &(slot, phys, total, element, writable)) in mappings.iter().enumerate() { + if slot >= 31 || mappings[..i].iter().any(|&(other, ..)| other == slot) { + return Err(EINVAL); + } + let entry = IO_MAPPINGS + slot * IO_MAPPING_SIZE; + let virt = entry + offset_of!(raw::IOMapping, virt_addr); + // The same register block as the manager's table. + let same_block = cfg.io_mappings.get(slot).and_then(|m| m.as_ref()).is_some_and(|m| { + let base = match (experiment, t6031) { + (Some(e), _) => e.io_block(slot, m.base as u64), + (None, Some(e)) => e.io_block(slot, m.base as u64), + (None, None) => m.base as u64, + }; + base & !0x3fff == phys & !0x3fff && m.writable == writable + }); + let covered = iomaps.iter().any(|io| { + io.slot == slot + && io.pointer_field() == Ok(virt) + && io.covers(phys, total) + }); + if !same_block || !covered || element == 0 || total % element != 0 { + dev_err!(dev, "M3: IO mapping slot {} ({:#x}) does not match the object layout\n", slot, phys); + return Err(EINVAL); + } + write(hwdata, entry + offset_of!(raw::IOMapping, phys_addr), &phys.to_le_bytes())?; + write(hwdata, entry + offset_of!(raw::IOMapping, total_size), &total.to_le_bytes())?; + write(hwdata, entry + offset_of!(raw::IOMapping, element_size), &element.to_le_bytes())?; + write( + hwdata, + entry + offset_of!(raw::IOMapping, readwrite), + &u64::from(writable).to_le_bytes(), + )?; + } + // Every IO mapping the runtime makes must belong to one of the entries above. + for io in iomaps.iter() { + if !mappings.iter().any(|&(slot, ..)| slot == io.slot) { + dev_err!(dev, "M3: IO mapping slot {} ({:#x}) has no device-tree InitData entry\n", io.slot, io.physical); + return Err(EINVAL); + } + } + Ok(()) +} + +pub(crate) fn check_upload( + dev: &device::Device, + firmware: &Firmware, + iomaps: &[storage::IoMap], + objects: &mut [Buffer], +) -> Result { + let mut bad = 0u32; + let version = objects.get_mut(INITDATA).ok_or(EINVAL)?.read_u64(0)?; + if version != firmware.initdata_magic { + dev_err!( + dev, + "M3: InitData self-check: version {:#x}, the loaded firmware expects {:#x}\n", + version, + firmware.initdata_magic + ); + bad += 1; + } + for (owner, offset, target, at) in RECORD_POINTERS { + let expected = objects.get(target).ok_or(EINVAL)?.va() + at as u64; + let value = objects.get_mut(owner).ok_or(EINVAL)?.read_u64(offset)?; + if value != expected { + dev_err!( + dev, + "M3: InitData self-check: owner {} +{:#x} holds {:#x}, not owner {} +{:#x} ({:#x})\n", + owner, + offset, + value, + target, + at, + expected + ); + bad += 1; + } + } + let hwdata = objects.get_mut(HWDATA).ok_or(EINVAL)?; + for slot in 0..IO_MAPPING_SLOTS { + let entry = IO_MAPPINGS + slot * IO_MAPPING_SIZE; + let phys = hwdata.read_u64(entry + offset_of!(raw::IOMapping, phys_addr))?; + let virt = hwdata.read_u64(entry + offset_of!(raw::IOMapping, virt_addr))?; + let total = u64::from(hwdata.read_u32(entry + offset_of!(raw::IOMapping, total_size))?); + let ok = match iomaps.iter().find(|io| io.slot == slot) { + Some(io) => { + phys == io.physical + io.offset as u64 + && virt == io.address + io.offset as u64 + && total != 0 + && phys + total <= io.physical + io.size as u64 + } + None => phys == 0 && virt == 0, + }; + if !ok { + dev_err!( + dev, + "M3: InitData self-check: IO mapping slot {} describes {:#x}+{:#x} at {:#x}, which the runtime does not map there\n", + slot, + phys, + total, + virt + ); + bad += 1; + } + } + let region = hwdata.read_u64(storage::GPU_REGION_PHYSICAL)?; + if region != firmware.resources.regions[0].base { + dev_err!( + dev, + "M3: InitData self-check: GPU region {:#x}, the resources admitted {:#x}\n", + region, + firmware.resources.regions[0].base + ); + bad += 1; + } + if bad != 0 { + dev_err!(dev, "M3: InitData self-check failed ({} findings); not publishing the InitData\n", bad); + return Err(EINVAL); + } + dev_info!( + dev, + "M3: InitData self-check passed: version {:#x}, {} record pointers, {} IO mappings, GPU region {:#x}\n", + version, + RECORD_POINTERS.len(), + iomaps.len(), + region + ); + Ok(()) +} + +/// The whole part of a positive `F32`, for logs. +fn whole(v: F32) -> u32 { + let bits = v.to_bits(); + match ((bits >> 23) & 0xff) as i32 - 127 { + e if e < 0 => 0, + e if e > 31 => u32::MAX, + e if e > 23 => ((bits & 0x7f_ffff) | 0x80_0000) << (e - 23), + e => ((bits & 0x7f_ffff) | 0x80_0000) >> (23 - e), + } +} + +/// The boot loader's decoded GPU leakage fuse values (`Soc::leak_fuse`), logged when it exported +/// them. Returns them when the boot loader's switch is set. With the switch set and no two +/// valid values, refuses: the boot loader then used neither the fuse nor the stand-in. +fn leak_fuse(dev: &device::Device, lf: &crate::m3_soc::LeakFuse) -> Result> { + let chosen = kernel::of::chosen(); + let switch = chosen + .as_ref() + .and_then(|c| c.get_property::>(lf.switch).ok()) + .is_some_and(|v| v.as_slice() == b"1\0"); + let values: Option> = chosen.as_ref().and_then(|c| c.get_property(lf.values).ok()); + // Positive, finite, nonzero. + let valid = |v: &F32| { + let b = v.to_bits(); + b != 0 && b >> 31 == 0 && (b >> 23) & 0xff != 0xff + }; + let pair = values + .as_ref() + .filter(|v| v.len() == 2 && v.iter().all(valid)) + .map(|v| [v[0], v[1]]); + let names = (lf.values.to_str().unwrap_or("?"), lf.switch.to_str().unwrap_or("?")); + match (pair, switch) { + (Some([a, b]), true) => { + dev_info!( + dev, + "M3: GPU leakage from the boot loader's fuse values: {} and {} (/chosen/{}, {} set)\n", + whole(a), + whole(b), + names.0, + names.1 + ); + Ok(Some([a, b])) + } + (Some([a, b]), false) => { + dev_info!( + dev, + "M3: the boot loader's GPU leakage fuse values are {} and {} (/chosen/{}); not used: {} is not set\n", + whole(a), + whole(b), + names.0, + names.1 + ); + Ok(None) + } + (None, true) => { + dev_err!( + dev, + "M3: {} is set, but /chosen/{} does not hold two valid values; GPU startup disabled\n", + names.1, + names.0 + ); + Err(EINVAL) + } + (None, false) => { + if values.is_some() { + dev_warn!(dev, "M3: /chosen/{} is malformed; ignored\n", names.0); + } + Ok(None) + } + } +} + +/// A zeroed image of owner `index`. +fn zeroed(index: usize) -> Result> { + let size = storage::allocation(index).map_err(|_| EINVAL)?.size; + Ok(KVVec::from_elem(0u8, size, GFP_KERNEL)?) +} + +/// Build the images of the generated owners from the device tree. +fn build_images( + dev: &device::Device, + firmware: &Firmware, + soc: &Soc, + experiment: Option<&Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, + iomaps: &[storage::IoMap], +) -> Result>>> { + check_layout().inspect_err(|_| { + dev_err!(dev, "M3: the constructed InitData records do not match the G15 InitData structures\n") + })?; + let cfg: &'static hw::HwConfig = match t6031 { + Some(e) => e.hwcfg(), + None => soc.hwcfg.ok_or(ENODEV)?, + }; + let io_mappings = match experiment { + Some(e) => e.io_mappings(), + None => match t6031 { + Some(e) => e.io_mappings(), + None => soc.io_mappings.ok_or(ENODEV)?, + }, + }; + let runtime_hwdata_b = match experiment { + Some(e) => e.hwdata_b(), + None => match t6031 { + Some(e) => e.hwdata_b(), + None => soc.hwdata_b.ok_or(ENODEV)?, + }, + }; + let mut pwr = hw::PwrConfig::load(dev, cfg).inspect_err(|e| { + dev_err!(dev, "M3: cannot read the GPU power configuration from the device tree ({:?})\n", e) + })?; + // The boot loader's decoded leakage fuse, when its switch makes it this boot's leakage: value + // 1 is the core leakage coefficient (the boot loader then writes it as apple,core-leak-coef + // too), value 2 goes to HwDataA's third leakage table below. + let fuse = match soc.leak_fuse { + Some(lf) => leak_fuse(dev, &lf)?, + None => None, + }; + if let Some([core, _]) = fuse { + match pwr.core_leak_coef.first_mut() { + Some(c) if c.to_bits() != core.to_bits() => { + dev_info!( + dev, + "M3: core leakage {} replaces the device tree's {}\n", + whole(core), + whole(*c) + ); + *c = core; + } + Some(_) => {} + None => return Err(EINVAL), + } + } + // A SoC whose power model is still a stand-in caps the power target: every operating point's + // power is scaled by the same factor, so the relative powers stay the boot loader's. + let power_cap_mw = match t6031 { + Some(e) => Some(e.power_cap_mw()), + None => soc.power_target_cap_mw, + }; + if let Some(cap) = power_cap_mw { + let max = pwr.max_power_mw; + if max > cap { + for ps in pwr.perf_states.iter_mut() { + ps.pwr_mw = (u64::from(ps.pwr_mw) * u64::from(cap) / u64::from(max)) as u32; + } + pwr.max_power_mw = pwr.perf_states.iter().map(|ps| ps.pwr_mw).max().unwrap_or(cap); + dev_info!( + dev, + "M3: GPU power target capped at {} mW (the device tree's operating points reach {} mW)\n", + pwr.max_power_mw, + max + ); + } + } + let node = dev.of_node().ok_or(ENODEV)?; + let dyncfg = hw::DynConfig { + uat_ttb_base: firmware.resources.regions[0].base, + id: soc_identity(soc, cfg, t6031.and_then(|e| e.revision_id())), + pwr, + firmware_version: node + .get_property::>(c_str!("apple,firmware-version")) + .unwrap_or_default(), + hw_data_a: KVVec::new(), + hw_data_b: KVVec::new(), + hw_globals: KVVec::new(), + }; + let g15 = G15Options { + layout: G15Layout::Runtime, + split_pstates: true, + // The cap is applied to the uploaded objects, for either source (`publish_pstates`). + cap: None, + timestamp_base: Some(crate::agx_memory::TIMESTAMP_RANGE.start), + reference_ppm: true, + runtime_hwdata_b: Some(runtime_hwdata_b), + per_cluster_voltages: soc.per_cluster_voltages, + }; + let c = initdata::InitDataBuilderG15V14_8_3::g15_contents(cfg, &dyncfg, &g15).inspect_err(|e| { + dev_err!(dev, "M3: cannot build InitData from the device tree ({:?})\n", e) + })?; + + let mut images = KVec::new(); + for _ in 0..storage::COUNT { + images.push(None, GFP_KERNEL)?; + } + + // InitData: the magic of the identified firmware, the host-mapped allocations flag and the + // UAT description; the pointers are the constructed record's. + type InitData = raw::InitDataG15V14_8_3<'static>; + let mut root = zeroed(INITDATA)?; + write(&mut root, offset_of!(InitData, ver_info), &firmware.initdata_magic.to_le_bytes())?; + write(&mut root, offset_of!(InitData, unk_2c), &1u32.to_le_bytes())?; + write(&mut root, offset_of!(InitData, uat_page_size), &0x4000u16.to_le_bytes())?; + write(&mut root, offset_of!(InitData, uat_page_bits), &[14, 3])?; + for (i, level) in c.uat_levels.iter().enumerate() { + let at = offset_of!(InitData, uat_level_info) + i * size_of::(); + write(&mut root, at, initdata::raw_bytes(level))?; + } + images[INITDATA] = Some(root); + + // RuntimePointers: only its configuration words; everything else is a pointer. + type RuntimePointers = raw::RuntimePointersG15V14_8_3<'static>; + let mut rt = zeroed(RUNTIME_POINTERS)?; + write(&mut rt, offset_of!(RuntimePointers, unk_2d0), &g15.runtime_pointers_2d0().to_le_bytes())?; + write(&mut rt, offset_of!(RuntimePointers, cswitch_hist_ctl), &[0xff])?; + images[RUNTIME_POINTERS] = Some(rt); + + // HwData: HwDataB, then HwDataA. + let mut hwdata = zeroed(HWDATA)?; + write(&mut hwdata, 0, initdata::raw_bytes(&*c.hwdata_b))?; + // The T8122 start experiment's unit masks (`asahi.t8122_unit_mask_a`/`_b`). + if let Some(e) = experiment { + type B = raw::HwDataBG15V14_8_3; + let (a, b) = e.unit_masks(); + write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; + write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; + dev_info!(dev, "M3 G15G start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); + } else if let Some(e) = t6031 { + type B = raw::HwDataBG15V14_8_3; + let (a, b) = e.unit_masks(); + write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; + write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; + dev_info!(dev, "M3 G15C start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); + } + write(&mut hwdata, HWDATA_A, initdata::raw_bytes(&*c.hwdata_a))?; + // The system counter value at InitData creation, the base of the firmware's first power and + // energy interval. + let now: u64; + // SAFETY: reading the architectural counter has no side effects and is allowed at EL1/EL2. + unsafe { core::arch::asm!("mrs {}, cntpct_el0", out(reg) now, options(nomem, nostack, preserves_flags)) }; + write( + &mut hwdata, + HWDATA_A + offset_of!(raw::HwDataAG15V14_8_3, init_timestamp), + &now.to_le_bytes(), + )?; + // The SoC's MTR sensor masks, where its table gives them. + if let Some(m) = soc.mtr_masks { + type A = raw::HwDataAG15V14_8_3; + for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { + write(&mut hwdata, HWDATA_A + at, &m.fast_die.to_le_bytes())?; + } + write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &m.alarm.to_le_bytes())?; + dev_info!( + dev, + "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", + m.fast_die, + m.alarm + ); + } else if let Some(e) = t6031 { + let (fast, alarm) = e.mtr_masks(dev)?; + type A = raw::HwDataAG15V14_8_3; + for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { + write(&mut hwdata, HWDATA_A + at, &fast.to_le_bytes())?; + } + write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &alarm.to_le_bytes())?; + dev_info!(dev, "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", fast, alarm); + } + // The second leakage fuse value: the first entry of HwDataA's third leakage table (the first + // table holds the core coefficient, from the builder). + if let Some([_, second]) = fuse { + type A = raw::HwDataAG15V14_8_3; + let at = HWDATA_A + offset_of!(A, cluster_tables) + 2 * size_of::<[u32; 8]>(); + write(&mut hwdata, at, &second.to_bits().to_le_bytes())?; + } + // The SoC's own values for words the shared builder writes otherwise. An armed T6031 copies + // the T8122 words only when asahi.t6031_fw_words=t8122. + let (hwdata_words, globals_words): (&[(usize, u32)], &[(usize, u32)]) = match t6031 { + Some(e) if e.fw_words() => ( + &crate::m3_soc::T8122_HWDATA_WORDS, + &crate::m3_soc::T8122_GLOBALS_WORDS, + ), + Some(_) => (&[], &[]), + None => (soc.hwdata_words, soc.globals_words), + }; + for &(at, value) in hwdata_words { + write(&mut hwdata, at, &value.to_le_bytes())?; + } + // T8122 with the G15 14.8.3 firmware (the only image the runtime accepts there): the 20 + // words of crate::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. + if crate::t8122_start::is_t8122(soc) + && core::ptr::eq(firmware.layout, &crate::m3_firmware::T8122_LAYOUT) + && cfg.gpu_gen == hw::GpuGen::G15 + { + if m3_params::t8122_hwdata_418c() { + let a = hwdata + .get_mut(HWDATA_A..HWDATA_A + size_of::()) + .ok_or(EINVAL)?; + t8122_hwdata::write(a).map_err(|()| EINVAL)?; + dev_info!( + dev, + "M3 G15G: HwDataA +0x418c..+0x41db: wrote the 20 words (asahi.t8122_hwdata_418c=0 leaves them 0)\n" + ); + } else { + dev_info!( + dev, + "M3 G15G: HwDataA +0x418c..+0x41db left 0 (asahi.t8122_hwdata_418c=0)\n" + ); + } + } + // On T8122, log and require the two HwDataB words the firmware's power management depends + // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero. + if core::ptr::eq(soc, &crate::m3_soc::T8122) { + type B = raw::HwDataBG15V14_8_3; + let a38 = read_u32(&hwdata, offset_of!(B, unk_460))?; + let a40 = read_u32(&hwdata, offset_of!(B, unk_468))?; + dev_info!(dev, "M3: HwDataB +0xa38 {:#x}, +0xa40 {:#x}\n", a38, a40); + if a38 == 0 || a40 == 0 { + dev_err!( + dev, + "M3: HwDataB +0xa38 or +0xa40 is 0; the firmware would not power the GPU cores\n" + ); + return Err(EINVAL); + } + } + fill_io_mappings(dev, cfg, experiment, t6031, io_mappings, iomaps, &mut hwdata)?; + images[HWDATA] = Some(hwdata); + + let mut globals = zeroed(GLOBALS)?; + write(&mut globals, 0, initdata::raw_bytes(&*c.globals))?; + // The gate of the firmware's frequency-feedback cap, per SoC. + let (ut, given) = m3_params::ut_engagement(soc); + write(&mut globals, offset_of!(raw::GlobalsG15V14_8_3, ut_engagement), &ut.to_le_bytes())?; + for &(at, value) in globals_words { + write(&mut globals, at, &value.to_le_bytes())?; + } + if !hwdata_words.is_empty() || !globals_words.is_empty() { + dev_info!( + dev, + "M3: {} HwData and {} Globals words set from the {} table\n", + hwdata_words.len(), + globals_words.len(), + soc.name + ); + } + if ut != 1 || given { + dev_info!( + dev, + "M3: Globals ut_engagement={}{}\n", + ut, + if given { " (asahi.m3_ut_engagement)" } else { "" } + ); + } + if crate::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) + .map_err(|_|EINVAL)? { + dev_info!(dev,"M3: T6030 Globals debug word cleared at +0x2c\n"); + } + images[GLOBALS] = Some(globals); + + let mut power = zeroed(GLOBALS_POWER)?; + write(&mut power, 0, initdata::raw_bytes(&*c.power_ctl_block))?; + images[GLOBALS_POWER] = Some(power); + + let mut status = zeroed(STATUS)?; + write(&mut status, 0, initdata::raw_bytes(&*c.status_block))?; + images[STATUS] = Some(status); + + let top = c.tables.primary_state(&dyncfg.pwr, c.tables.len - 1); + dev_info!( + dev, + "M3: device-tree InitData: {} performance states in two voltage-sorted tables from {} device-tree states, top {} MHz at {} mV, SRAM floor {} mV, maximum power {} mW\n", + c.tables.len - 1, + dyncfg.pwr.perf_states.len() - 1, + top.freq_hz / 1_000_000, + top.max_volt_mv(), + dyncfg.pwr.min_sram_microvolt.div_ceil(1000), + dyncfg.pwr.max_power_mw + ); + log_tables(dev, &c.hwdata_b, c.tables.len); + Ok(images) +} + +/// Log the published performance-state tables, one line per entry: the primary table (fastest +/// state per voltage) and the secondary one (slowest), each with the device-tree index of the +/// entry, and the third table. +fn log_tables(dev: &device::Device, b: &raw::HwDataBG15V14_8_3, len: usize) { + let uniform = |row: &[u32; 8]| row.iter().all(|v| *v == row[0]); + for i in 1..len.min(16) { + let aux = &b.aux_ps_3; + let ok = uniform(&b.voltages[i]) && uniform(&b.voltages_sram[i]); + dev_info!( + dev, + "M3: performance state {}: primary dt {} {} MHz core {} mV SRAM {} mV; secondary dt {} {} MHz; third {} MHz core {} mV SRAM {} mV{}\n", + i, + b.unk_arr_0[i], + b.frequencies[i], + b.voltages[i][0], + b.voltages_sram[i][0], + b.unk_arr_0[16 + i], + b.frequencies_2[i], + aux.frequencies[i], + aux.voltages[i][0], + aux.voltages_sram[i][0], + if ok { "" } else { " (columns differ)" } + ); + } +} + +/// HwDataA words that hold the highest usable performance state scaled by 100. +const HWA_MAX_SCALED: [usize; 15] = { + type A = raw::HwDataAG15V14_8_3; + let se = offset_of!(A, unk_e10_0); + [ + offset_of!(A, max_pstate_scaled), + offset_of!(A, max_pstate_scaled_2), + offset_of!(A, max_pstate_scaled_3), + offset_of!(A, max_pstate_scaled_4), + offset_of!(A, max_pstate_scaled_5), + offset_of!(A, max_pstate_scaled_7), + offset_of!(A, max_pstate_scaled_8), + offset_of!(A, max_pstate_scaled_9), + offset_of!(A, max_pstate_scaled_10), + offset_of!(A, max_pstate_scaled_11), + offset_of!(A, max_pstate_scaled_12), + offset_of!(A, max_pstate_scaled_13), + offset_of!(A, max_pstate_scaled_14), + se + offset_of!(raw::HwDataA140Extra, max_pstate_scaled_1), + se + offset_of!(raw::HwDataA140Extra, max_pstate_scaled_2), + ] +}; + +/// HwDataA words that hold the base (lowest settled) performance state scaled by 100. +const HWA_BASE_SCALED: [usize; 4] = { + type A = raw::HwDataAG15V14_8_3; + [ + offset_of!(A, base_pstate_scaled), + offset_of!(A, base_pstate_scaled_2), + offset_of!(A, base_pstate_scaled_3), + offset_of!(A, base_pstate_scaled_4), + ] +}; + +/// Offsets in Globals of the runtime cap words: the second and third power-interface targets and +/// the performance-state cap, each the highest usable state scaled by 100. +const GLOBALS_RUNTIME_CAP: [usize; 3] = { + type G = raw::GlobalsG15V14_8_3; + let power_if_target = offset_of!(G, power_if_target); + [power_if_target + 4, power_if_target + 8, offset_of!(G, perf_state_cap)] +}; + +pub(crate) fn publish_pstates( + dev: &device::Device, + hwdata: &mut Buffer, + globals: &mut Buffer, + p: &PstatePolicy, +) -> Result { + use crate::float::F32; + type A = raw::HwDataAG15V14_8_3; + let a = |offset: usize| HWDATA_A + offset; + let full = 100 * p.table_max; + let base_scaled = hwdata.read_u32(a(offset_of!(A, base_pstate_scaled)))?; + let globals_max = GLOBALS_RUNTIME_CAP; + + let mut expected = true; + for off in HWA_MAX_SCALED { + expected &= hwdata.read_u32(a(off))? == full; + } + for off in HWA_BASE_SCALED { + expected &= hwdata.read_u32(a(off))? == base_scaled; + } + for off in globals_max { + expected &= globals.read_u32(off)? == full; + } + expected &= hwdata.read_u32(a(offset_of!(A, max_pstate_scaled_6)))? == F32::from(full).to_bits(); + let base = base_scaled / 100; + if !expected + || base_scaled % 100 != 0 + || base == 0 + || base > p.safe + || p.boot > p.safe + || p.safe > p.max + { + dev_err!(dev, "M3: unexpected performance-state words in the InitData; not starting the firmware\n"); + return Err(EINVAL); + } + + let ceiling = 100 * p.max; + let max_power = F32::from_bits(hwdata.read_u32(a(offset_of!(A, max_power_1)))?); + let min_duty = hwdata.read_u32(a(offset_of!(A, pwr_min_duty_cycle)))?; + for off in HWA_MAX_SCALED { + hwdata.u32(a(off), ceiling)?; + } + hwdata.u32(a(offset_of!(A, max_pstate_scaled_6)), F32::from(ceiling).to_bits())?; + hwdata.u32(a(offset_of!(A, pwr_pstate_related_k)), (-F32::from(ceiling) / max_power).to_bits())?; + hwdata.u32( + a(offset_of!(A, pwr_pstate_max_dc_offset)), + (min_duty as i32 - ceiling as i32) as u32, + )?; + hwdata.u32( + a(offset_of!(A, boost_state_unk_k)), + (F32::from(p.max - base) / crate::f32!(0.95)).to_bits(), + )?; + for off in globals_max { + globals.u32(off, 100 * p.safe)?; + } + + for off in [offset_of!(A, actual_pstate), offset_of!(A, tgt_pstate)] { + hwdata.u32(a(off), p.boot)?; + } + let settle = base; + dev_info!( + dev, + "M3: firmware performance request: state {} ({} MHz), base state {}, cap state {} ({} MHz) of 1..={}\n", + p.boot, + p.boot_mhz, + settle, + p.safe, + p.safe_mhz, + p.table_max + ); + if p.safe != p.max { + dev_info!( + dev, + "M3: firmware highest state {} ({} MHz); the runtime cap starts at {} ({} MHz)\n", + p.max, + p.max_mhz, + p.safe, + p.safe_mhz + ); + } + Ok(()) +} + +/// Set the runtime cap: the Globals runtime cap words, which the firmware keeps reading while it +/// runs, so the store alone moves the cap, with no message. `cap` must be a state of the +/// published table. +pub(crate) fn publish_runtime_cap(globals: &mut Buffer, cap: u32) -> Result { + if cap == 0 || cap >= 16 { + return Err(EINVAL); + } + for off in GLOBALS_RUNTIME_CAP { + globals.u32(off, 100 * cap)?; + } + crate::agx_memory::publish(); + Ok(()) +} + +/// The integer part of a non-negative, finite F32 given by its bits; None otherwise. +fn f32_bits_to_u32(bits: u32) -> Option { + let exp = (bits >> 23) & 0xff; + if bits >> 31 != 0 || exp == 0xff { + return None; + } + if exp < 127 { + return Some(0); + } + let mant = u64::from((bits & 0x7f_ffff) | 0x80_0000); + let shift = exp - 127; + let value = if shift >= 23 { + mant.checked_shl(shift - 23)? + } else { + mant >> (23 - shift) + }; + u32::try_from(value).ok() +} + +/// One reading of the performance state: SGX+0xe01000, and the HwDataA actual and current +/// state and frequency (MHz, F32 bits) the firmware publishes. +#[derive(Copy, Clone, PartialEq, Eq)] +struct PstateSample { + register: u32, + actual: u32, + current: u32, + freq_bits: u32, +} + +/// Number of lines logging changes of the performance state per boot. +const PSTATE_LOG_LINES: u32 = 64; + +/// Number of lines warning about a reported state above the runtime cap per boot. +const RUNTIME_CAP_WARN_LINES: u32 = 16; + +/// Checks the performance state the firmware reports against the ceiling and the runtime cap, +/// after boot and after every job. +/// +/// The firmware publishes its actual and current state and its frequency in HwDataA. A +/// frequency above the ceiling entry's, or, when no running frequency is reported, a state in +/// the published table above the ceiling, that is still there when read again 1 ms later fails +/// the check. The same above the runtime cap (when it is below the ceiling) is reported to the +/// caller, which decides whether it persists too long (the thermal limit). Values outside the +/// published table are logged and not enforced: they are not states of this table. +/// SGX+0xe01000 is logged next to them but not enforced, as the index space of that register is +/// not established on every board. The published tables list device-tree operating points only, +/// so the check guards the cap, not the voltages. +pub(crate) struct PstateWatch { + policy: PstatePolicy, + last: Option, + lines: u32, + warn_lines: u32, + register_noted: bool, + unknown_noted: bool, + excused_noted: bool, +} + +impl PstateWatch { + pub(crate) fn new(policy: PstatePolicy) -> Self { + PstateWatch { + policy, + last: None, + lines: 0, + warn_lines: 0, + register_noted: false, + unknown_noted: false, + excused_noted: false, + } + } + + fn sample(hwdata: &mut Buffer, register: u32) -> Result { + type A = raw::HwDataAG15V14_8_3; + Ok(PstateSample { + register, + actual: hwdata.read_u32(HWDATA_A + offset_of!(A, actual_pstate))?, + current: hwdata.read_u32(HWDATA_A + offset_of!(A, cur_pstate))?, + freq_bits: hwdata.read_u32(HWDATA_A + offset_of!(A, freq_mhz))?, + }) + } + + /// Frequency of entry `state` of the published table, in MHz. + fn mhz(&self, state: u32) -> u32 { + self.policy.freqs.get(state as usize).copied().unwrap_or(0) + } + + /// The reported frequency in MHz, if it is a running frequency of the published table. + fn freq_mhz(&self, s: &PstateSample) -> Option { + f32_bits_to_u32(s.freq_bits).filter(|mhz| *mhz > 0 && *mhz <= self.policy.table_max_mhz) + } + + /// Whether a reported state lies within the published table but above `cap`. + fn state_above(&self, s: &PstateSample, cap: u32) -> Option<(&'static str, u32)> { + let above = |v: u32| v > cap && v <= self.policy.table_max; + if above(s.actual) { + Some(("actual state", s.actual)) + } else if above(s.current) { + Some(("current state", s.current)) + } else { + None + } + } + + /// The reported value above `cap` that lies within the published table, if any. The + /// frequency is the physical quantity: a state above the cap is excused when the reported + /// frequency is a running frequency within the cap (the state words then use another + /// index space), and counts otherwise. + fn above(&self, s: &PstateSample, cap: u32) -> Option<(&'static str, u32)> { + match self.freq_mhz(s) { + Some(mhz) if mhz > self.mhz(cap) => Some(("frequency (MHz)", mhz)), + Some(_) => None, + None => self.state_above(s, cap), + } + } + + /// Whether a reported value is not a state or frequency of the published table. + fn unknown(&self, s: &PstateSample) -> bool { + let p = &self.policy; + s.actual > p.table_max + || s.current > p.table_max + || f32_bits_to_u32(s.freq_bits).is_none_or(|mhz| mhz > p.table_max_mhz) + } + + fn log(&self, dev: &device::Device, what: &str, s: &PstateSample, cap: u32) { + dev_info!( + dev, + "M3: performance state {}: firmware actual {} current {} frequency {} MHz, sgx+0xe01000 {:#x}; cap state {} ({} MHz)\n", + what, + s.actual, + s.current, + f32_bits_to_u32(s.freq_bits).unwrap_or(u32::MAX), + s.register, + cap, + self.mhz(cap) + ); + } + + /// Read the existing firmware state words and SGX state register without + /// requesting a state, publishing a cap, or enforcing a sample. + pub(crate) fn log_active(&self, dev: &device::Device, hwdata: &mut Buffer, + register: u32, cap: u32, sequence: u32) -> Result { + let s = Self::sample(hwdata, register)?; + dev_info!(dev, + "M3_ACTIVE_POWER sample={} actual={} current={} freq_bits={:#x} sgx={:#x} cap={} cap_mhz={}\n", + sequence, s.actual, s.current, s.freq_bits, s.register, cap, self.mhz(cap)); + Ok(()) + } + + /// Check the reported performance state. `register` reads SGX+0xe01000; `what` names the + /// point of the check for the log; `cap` is the runtime cap the firmware may use now + /// (clamped to the ceiling). Returns ERANGE when the firmware stays above the ceiling; + /// otherwise whether it stays above `cap`, and the highest state it reports (0 if neither + /// state word is a state of the published table). + pub(crate) fn check( + &mut self, + dev: &device::Device, + hwdata: &mut Buffer, + register: impl Fn() -> Result, + what: &str, + cap: u32, + ) -> Result<(bool, u32)> { + let ceiling = self.policy.max; + let cap = cap.clamp(1, ceiling); + let mut s = Self::sample(hwdata, register()?)?; + let mut over = false; + if let Some((field, value)) = self.above(&s, cap) { + if cap == ceiling || self.warn_lines < RUNTIME_CAP_WARN_LINES { + self.warn_lines = self.warn_lines.saturating_add(u32::from(cap != ceiling)); + dev_warn!( + dev, + "M3: firmware reports {} {} above the cap {} ({} MHz) {}; reading again\n", + field, + value, + cap, + self.mhz(cap), + what + ); + } + fsleep(Delta::from_millis(1)); + s = Self::sample(hwdata, register()?)?; + if let Some((field, value)) = self.above(&s, ceiling) { + self.log(dev, what, &s, cap); + dev_err!( + dev, + "M3: firmware {} {} stays above the cap {} ({} MHz); marking the GPU failed\n", + field, + value, + ceiling, + self.mhz(ceiling) + ); + return Err(ERANGE); + } + over = self.above(&s, cap).is_some(); + } + if !self.excused_noted { + if let Some((field, value)) = self.state_above(&s, cap) { + if !over { + self.excused_noted = true; + dev_info!( + dev, + "M3: firmware {} {} is above the cap {} but its frequency is within it {}; not enforced\n", + field, + value, + cap, + what + ); + } + } + } + if !self.unknown_noted && self.unknown(&s) { + self.unknown_noted = true; + dev_info!( + dev, + "M3: firmware performance words outside the published table (states 1..={}, up to {} MHz) {}; not enforced\n", + self.policy.table_max, + self.policy.table_max_mhz, + what + ); + self.log(dev, what, &s, cap); + } + if !self.register_noted && s.register & 0xf > cap { + self.register_noted = true; + dev_info!( + dev, + "M3: sgx+0xe01000 reads {:#x}, above the cap as a table index; logged only\n", + s.register + ); + } + let key = PstateSample { + register: s.register & 0xf, + ..s + }; + if self.last != Some(key) { + if self.lines < PSTATE_LOG_LINES { + self.log(dev, what, &s, cap); + } else if self.lines == PSTATE_LOG_LINES { + dev_info!(dev, "M3: further performance state changes are not logged\n"); + } + self.lines = self.lines.saturating_add(1); + self.last = Some(key); + } + let peak = [s.actual, s.current] + .into_iter() + .filter(|v| *v <= self.policy.table_max) + .max() + .unwrap_or(0); + Ok((over, peak)) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs new file mode 100644 index 00000000000000..5ecfd820b4e4f1 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Board facts for the M3 runtime, read from the hardware and the device tree instead of being +//! fixed to one board. The per-SoC facts are in [`crate::m3_soc`]. + +use kernel::{bindings, c_str, device, device::Core, io::resource::Resource, of, platform, prelude::*, uapi}; + +use crate::m3_resources::{ + reserved_resource, + Region, + Resources, // +}; +use crate::m3_soc::Soc; + +/// Whether the root node's compatible list contains `compatible`. +fn board_is(compatible: &[u8]) -> bool { + let Some(root) = kernel::of::root() else { + return false; + }; + let Ok(board) = root.get_property::>(c_str!("compatible")) else { + return false; + }; + board.split(|b| *b == 0).any(|s| s == compatible) +} + +/// Whether this is a board the M3 runtime is validated on (`soc.validated_boards`). +/// `asahi.m3_expose=auto` registers the render node and `asahi.m3_backend=auto` starts the +/// runtime only on these. +pub(crate) fn runtime_validated_board(soc: &Soc) -> bool { + soc.validated_boards.iter().any(|board| board_is(board)) +} + +/// The number of core slots of `soc` (2 x 10 on T6030, 4 x 10 on T6031). +pub(crate) fn core_slots(soc: &Soc) -> u32 { + soc.cores_per_cluster * soc.clusters +} + +/// Whether the core-enable mask describes a usable GPU of `soc`: at least one core, and no core +/// outside its core slots. The mask is the fused core-enable words from SGX+0xe01500, the second +/// word as bits 32..63 when the SoC has more than 32 core slots. It varies with the SKU (0x6f5fc +/// on a 14-core T6030, 0x7fdff on an 18-core one); absent cores must never be enabled. +pub(crate) fn core_mask_valid(soc: &Soc, mask: u64) -> bool { + let slots = core_slots(soc); + mask != 0 && slots <= 64 && (slots == 64 || mask >> slots == 0) +} + +/// Per-cluster UAPI core masks for a core-enable mask of `soc`: consecutive fields of +/// `cores_per_cluster` bits, cluster 0 lowest, across the word boundary (the packing the +/// identification code uses for every multi-word mask, `regs.rs`). +pub(crate) fn core_masks(soc: &Soc, mask: u64) -> [u32; uapi::DRM_ASAHI_MAX_CLUSTERS as usize] { + let mut masks = [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize]; + let width = soc.cores_per_cluster.min(32); + let field = (1u64 << width) - 1; + for (cluster, slot) in masks.iter_mut().enumerate().take(soc.clusters as usize) { + let shift = cluster as u32 * soc.cores_per_cluster; + *slot = mask.checked_shr(shift).map_or(0, |m| (m & field) as u32); + } + masks +} + +/// The highest GPU frequency in the device tree OPP table, in kHz, or None when the GPU node +/// has no populated OPP table. +pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { + let node = pdev.as_ref().of_node()?; + let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; + let mut max_hz: u64 = 0; + for opp in opps.children() { + if let Ok(hz) = opp.get_property::(c_str!("opp-hz")) { + max_hz = max_hz.max(hz); + } + } + u32::try_from(max_hz / 1000).ok().filter(|khz| *khz != 0) +} + +/// The shape of the performance-state table the device tree's operating points give: states +/// above the off state and the highest frequency in MHz (`t8122_admission::opp_table_shape`). +/// Every cluster of a point must have the same voltage, except on a SoC with per-cluster +/// voltages, where the highest `opp-microvolt` cell orders the point. None without a readable +/// OPP table. +pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Option<(u32, u32)> { + let node = pdev.as_ref().of_node()?; + let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; + let mut points = KVec::new(); + for opp in opps.children() { + let hz = opp.get_property::(c_str!("opp-hz")).ok()?; + let microvolt = opp.get_property::>(c_str!("opp-microvolt")).ok()?; + let mv = microvolt.first()?.div_ceil(1000); + if microvolt.len() != soc.clusters as usize + || (!soc.per_cluster_voltages && microvolt.iter().any(|uv| uv.div_ceil(1000) != mv)) + { + return None; + } + // With per-cluster voltages a point is ordered by its highest cluster voltage, as the + // published tables are (`initdata::PStateTables`). + let uv = if soc.per_cluster_voltages { + *microvolt.iter().max()? + } else { + *microvolt.first()? + }; + points.push((hz, uv), GFP_KERNEL).ok()?; + } + crate::t8122_admission::opp_table_shape(points) +} + +/// Register windows the runtime maps on `soc`: name, base, minimum size. The runtime uses the +/// first 16 KiB of `asc` (`m3_device`); the first 16 MiB of `sgx` cover the ID block at +/// +0xd04000, the Fender MMU block at +0xd08000 and the core masks at +0xe01500. +fn reg_windows(soc: &Soc) -> [(&'static CStr, u64, u64); 2] { + [ + (c_str!("asc"), soc.windows.asc, 0x4000), + (c_str!("sgx"), soc.windows.sgx, 0x100_0000), + ] +} + +/// The `reg` cells (two address and two size cells) of the mailbox of `soc`: 16 KiB at its base. +fn expected_mailbox_reg(soc: &Soc) -> [u32; 4] { + [(soc.windows.mailbox >> 32) as u32, soc.windows.mailbox as u32, 0, 0x4000] +} + +/// Reserved regions of the firmware handoff, in `Resources::regions` order. +const REGIONS: [&CStr; 6] = [ + c_str!("ttbs"), + c_str!("pagetables"), + c_str!("handoff"), + c_str!("shared-l2"), + c_str!("fw-text"), + c_str!("fw-data"), +]; + +/// Static reserved-memory nodes that describe a handoff region when the GPU node does not list +/// it under its own name: the bootloader fills these nodes (the UAT regions of the firmware's +/// page-table handoff) whether or not the GPU node references them. The second-level table of +/// the firmware's upper page-table tree ("shared-l2") is reserved this way without being in the +/// GPU node's memory-region list. +const REGION_NODES: [(&CStr, &CStr); 4] = [ + (c_str!("ttbs"), c_str!("/reserved-memory/uat-ttbs")), + (c_str!("pagetables"), c_str!("/reserved-memory/uat-pagetables")), + (c_str!("handoff"), c_str!("/reserved-memory/uat-handoff")), + (c_str!("shared-l2"), c_str!("/reserved-memory/uat-pagetables-l2")), +]; + +/// Properties that describe the loaded firmware's segments on the GPU node. +const SEGMENT_PROPS: [&CStr; 3] = [ + c_str!("apple,m3-handoff-version"), + c_str!("apple,firmware-segment-vas"), + c_str!("apple,firmware-segment-flags"), +]; + +/// Power-management properties of a GPU node whose SoC takes them from the boot loader +/// (`Soc::power_from_boot_loader`), one 32-bit cell each: the ones the T6030 device tree carries +/// as static values, and the shader-engine target. The boot loader copies each from the ADT's GPU +/// node (`apple,X` from `gpu-X`; `apple,se-target` from `gpu-se-tgt`). +/// +/// `apple,idleoff-standby-timer` is not among them: the J613 ADT has no +/// `gpu-idleoff-standby-timer`, so the boot loader adds it only when the ADT has one, and the +/// HwConfig's default applies otherwise. +const BOOT_LOADER_POWER: [&CStr; 40] = [ + c_str!("apple,perf-base-pstate"), + c_str!("apple,min-sram-microvolt"), + c_str!("apple,avg-power-filter-tc-ms"), + c_str!("apple,avg-power-ki-only"), + c_str!("apple,avg-power-kp"), + c_str!("apple,avg-power-min-duty-cycle"), + c_str!("apple,avg-power-target-filter-tc"), + c_str!("apple,fast-die0-integral-gain"), + c_str!("apple,fast-die0-proportional-gain"), + c_str!("apple,perf-boost-ce-step"), + c_str!("apple,perf-boost-min-util"), + c_str!("apple,perf-filter-drop-threshold"), + c_str!("apple,perf-filter-time-constant"), + c_str!("apple,perf-filter-time-constant2"), + c_str!("apple,perf-integral-gain"), + c_str!("apple,perf-integral-gain2"), + c_str!("apple,perf-integral-min-clamp"), + c_str!("apple,perf-proportional-gain"), + c_str!("apple,perf-proportional-gain2"), + c_str!("apple,perf-tgt-utilization"), + c_str!("apple,power-sample-period"), + c_str!("apple,ppm-filter-time-constant-ms"), + c_str!("apple,ppm-ki"), + c_str!("apple,ppm-kp"), + c_str!("apple,pwr-filter-time-constant"), + c_str!("apple,pwr-integral-gain"), + c_str!("apple,pwr-integral-min-clamp"), + c_str!("apple,pwr-min-duty-cycle"), + c_str!("apple,pwr-proportional-gain"), + c_str!("apple,pwr-sample-period-aic-clks"), + c_str!("apple,se-engagement-criteria"), + c_str!("apple,se-filter-time-constant"), + c_str!("apple,se-filter-time-constant-1"), + c_str!("apple,se-inactive-threshold"), + c_str!("apple,se-ki"), + c_str!("apple,se-ki-1"), + c_str!("apple,se-kp"), + c_str!("apple,se-kp-1"), + c_str!("apple,se-reset-criteria"), + c_str!("apple,se-target"), +]; + +/// Check that the boot loader gave a GPU node of `soc` every per-machine property the runtime +/// would otherwise take from a static device tree or a driver default: [`BOOT_LOADER_POWER`], the +/// 64-bit `apple,fast-die0-sensor-mask`, one core and one SRAM leakage coefficient per cluster, +/// the firmware version (three cells, not all zero), and an operating-point table of at least two +/// entries, each with one voltage per cluster and a power, and a frequency after the first. Logs +/// every missing or malformed property. +fn check_boot_loader_power(dev: &device::Device, node: &of::Node, soc: &Soc) -> Result { + let mut bad = 0u32; + let mut expect = |name: &CStr, want: usize, nonzero: bool| { + match node.get_property::>(name) { + Ok(value) if value.len() == want && (!nonzero || value.iter().any(|b| *b != 0)) => {} + Ok(value) => { + dev_info!( + dev, + "M3: not admitted: {:?} is {} bytes, {} expected{}\n", + name, + value.len(), + want, + if nonzero { ", not all zero" } else { "" } + ); + bad += 1; + } + Err(_) => { + dev_info!(dev, "M3: not admitted: the boot loader did not add {:?}\n", name); + bad += 1; + } + } + }; + for name in BOOT_LOADER_POWER { + expect(name, 4, false); + } + expect(c_str!("apple,fast-die0-sensor-mask"), 8, false); + expect(c_str!("apple,core-leak-coef"), 4 * soc.clusters as usize, false); + expect(c_str!("apple,sram-leak-coef"), 4 * soc.clusters as usize, false); + expect(c_str!("apple,firmware-version"), 12, true); + + let mut opps = 0u32; + let mut opps_ok = true; + match node.parse_phandle(c_str!("operating-points-v2"), 0) { + Some(table) => { + for opp in table.children() { + let hz = opp.get_property::(c_str!("opp-hz")).unwrap_or(0); + let volts = opp.get_property::>(c_str!("opp-microvolt")).map_or(0, |v| v.len()); + let power = opp.get_property::(c_str!("opp-microwatt")).is_ok(); + opps_ok &= volts == soc.clusters as usize && power && (opps == 0 || hz != 0); + opps += 1; + } + } + None => opps_ok = false, + } + if !opps_ok || opps < 2 { + dev_info!( + dev, + "M3: not admitted: the operating points are not the boot loader's ({} entries; each needs opp-hz, {} opp-microvolt cells and opp-microwatt)\n", + opps, + soc.clusters + ); + bad += 1; + } + if bad != 0 { + return Err(EINVAL); + } + dev_info!(dev, "M3: boot loader power configuration accepted ({} operating points)\n", opps); + Ok(()) +} + +/// Whether the GPU node lists the memory region `name` in memory-region-names. +fn lists_region(node: &of::Node, name: &CStr) -> bool { + node.get_property::>(c_str!("memory-region-names")) + .is_ok_and(|names| names.split(|b| *b == 0).any(|n| n == name.to_bytes())) +} + +/// The static reserved-memory node that stands in for the handoff region `name`, if the GPU +/// node does not list the region itself and the device tree has such a node. +fn region_node_path(node: &of::Node, name: &CStr) -> Option<&'static CStr> { + if lists_region(node, name) { + return None; + } + REGION_NODES + .iter() + .find(|(region, _)| region.to_bytes() == name.to_bytes()) + .map(|(_, path)| *path) +} + +/// A reference to a device tree node found by path, released on drop. +struct NodeRef(*mut bindings::device_node); + +impl NodeRef { + fn find(path: &CStr) -> Option { + // SAFETY: `path` is a NUL-terminated string; the lookup takes a node reference or + // returns NULL. + let np = unsafe { bindings::of_find_node_opts_by_path(path.as_char_ptr(), core::ptr::null_mut()) }; + (!np.is_null()).then_some(NodeRef(np)) + } +} + +impl Drop for NodeRef { + fn drop(&mut self) { + // SAFETY: the reference was taken by `find`. + unsafe { bindings::of_node_put(self.0) }; + } +} + +/// Look up the handoff region `name` through its static reserved-memory node (see +/// [`REGION_NODES`]). Returns None when the GPU node lists the region itself or the device tree +/// has no such node; otherwise the region, which must be an enabled reserved-memory node the +/// kernel reserved at boot. Returns whether the node is no-map as well. +pub(crate) fn static_region(node: &of::Node, name: &CStr) -> Option> { + let path = region_node_path(node, name)?; + let np = NodeRef::find(path)?; + // SAFETY: `np` holds a node reference for the duration of these read-only queries. + let (available, rmem, nomap) = unsafe { + ( + bindings::of_device_is_available(np.0), + bindings::of_reserved_mem_lookup(np.0), + bindings::of_property_read_bool(np.0, c_str!("no-map").as_char_ptr()), + ) + }; + if !available || rmem.is_null() { + return Some(Err(EINVAL)); + } + // SAFETY: the reserved-memory table entry lives as long as the kernel. + let (base, size) = unsafe { ((*rmem).base, (*rmem).size) }; + if base == 0 || size == 0 || base.checked_add(size).is_none_or(|end| end > 1 << 42) { + return Some(Err(EINVAL)); + } + let raw = bindings::resource { + start: base, + end: base + size - 1, + flags: bindings::IORESOURCE_MEM as _, + ..Default::default() + }; + // SAFETY: Resource is repr(transparent) over Opaque. The descriptor is owned and + // has no pointers into temporary data; its range is checked above. + Some(Ok((unsafe { core::mem::transmute::(raw) }, nomap))) +} + +/// Admit a GPU of `soc` described by the device tree, either statically or by a runtime overlay. +/// +/// The device must be a GPU node of `soc` with the ASC and SGX windows at their addresses +/// (at least as large as the runtime maps), the GPU coprocessor mailbox, the six handoff regions +/// by name (from the reserved-memory registry or no-map overlay nodes), and the firmware segment +/// description. Every refusal is logged. +pub(crate) fn admit(pdev: &platform::Device, soc: &Soc) -> Result { + let dev = pdev.as_ref(); + let refuse = |what: &str, err: Error| { + dev_info!(dev, "M3: not admitted: {}\n", what); + err + }; + let node = dev.of_node().ok_or(ENODEV)?; + + if !board_is(soc.board.as_bytes()) { + dev_info!(dev, "M3: not admitted: not a {} board\n", soc.name); + return Err(ENODEV); + } + let compatible: KVec = node.get_property(c_str!("compatible"))?; + if !compatible.split(|b| *b == 0).any(|s| s == soc.gpu.as_bytes()) { + dev_info!(dev, "M3: not admitted: GPU node is not {}\n", soc.gpu); + return Err(ENODEV); + } + // The runtime runs the firmware that the bootloader loaded, in place: without the + // bootloader's description of its segments there is nothing to admit. + if !lists_region(&node, c_str!("fw-text")) + || !lists_region(&node, c_str!("fw-data")) + || SEGMENT_PROPS.iter().any(|p| node.get_property::>(p).is_err()) + { + return Err(refuse( + "the device tree does not describe the loaded GPU firmware segments (fw-text/fw-data regions and apple,firmware-segment-* properties); the bootloader predates this", + ENODEV, + )); + } + dev_info!(dev, "M3: board compatible accepted\n"); + if soc.power_from_boot_loader { + check_boot_loader_power(dev, &node, soc)?; + } + + for name in REGIONS { + if lists_region(&node, name) { + continue; + } + match region_node_path(&node, name) { + Some(path) if NodeRef::find(path).is_some() => { + dev_info!(dev, "M3: {:?} region: static node {:?}\n", name, path) + } + _ => { + dev_info!(dev, "M3: not admitted: no {:?} memory region\n", name); + return Err(EINVAL); + } + } + } + dev_info!(dev, "M3: memory region list accepted\n"); + + for (name, base, min_size) in reg_windows(soc) { + let res = pdev + .resource_by_name(name) + .ok_or_else(|| refuse("missing register window", EINVAL))?; + if res.start() != base || res.size() < min_size { + dev_info!( + dev, + "M3: not admitted: {:?} window {:#x}+{:#x}, need {:#x}+{:#x} or larger\n", + name, + res.start(), + res.size(), + base, + min_size + ); + return Err(EINVAL); + } + } + dev_info!(dev, "M3: register resources accepted\n"); + + let mboxes: KVec = node.get_property(c_str!("mboxes"))?; + let mbox = (mboxes.len() == 1) + .then(|| node.parse_phandle(c_str!("mboxes"), 0)) + .flatten() + .ok_or_else(|| refuse("expected exactly one mailbox", EINVAL))?; + let compat: KVec = mbox.get_property(c_str!("compatible"))?; + let mbox_reg: KVec = mbox.get_property(c_str!("reg"))?; + let irq_names: KVec = mbox.get_property(c_str!("interrupt-names"))?; + let irqs: KVec = mbox.get_property(c_str!("interrupts"))?; + let cells: u32 = mbox.get_property(c_str!("#mbox-cells"))?; + if !soc.mailbox_compatibles.iter().any(|c| *c == compat.as_slice()) + || mbox_reg.as_slice() != expected_mailbox_reg(soc) + || cells != 0 + || irq_names.as_slice() != b"send-empty\0send-not-empty\0recv-empty\0recv-not-empty\0" + || irqs.as_slice() != soc.mailbox_interrupts + { + return Err(refuse("unexpected GPU mailbox description", EINVAL)); + } + dev_info!(dev, "M3: mailbox resource accepted\n"); + + let mut regions = [Region { base: 0, size: 0 }; 6]; + for (i, name) in REGIONS.iter().enumerate() { + let res = reserved_resource(&node, name).map_err(|e| { + dev_info!(dev, "M3: not admitted: {:?} region unusable ({:?})\n", name, e); + e + })?; + regions[i] = Region { + base: res.start(), + size: res.size(), + }; + } + dev_info!(dev, "M3: reserved resources {:?}\n", regions); + + let version: u32 = node + .get_property(c_str!("apple,m3-handoff-version")) + .map_err(|e| refuse("no firmware handoff description", e))?; + let vas: KVec = node.get_property(c_str!("apple,firmware-segment-vas"))?; + let flags: KVec = node.get_property(c_str!("apple,firmware-segment-flags"))?; + Resources::validate( + version, + regions, + vas.as_slice().try_into().map_err(|_| EINVAL)?, + flags.as_slice().try_into().map_err(|_| EINVAL)?, + ) + .map_err(|e| { + dev_info!(dev, "M3: not admitted: handoff description rejected ({:?})\n", e); + EINVAL + }) +} + +/// Whether the GPU node links to a PMP instance (`apple,pmp`) through which the GPU power vote +/// is cast. Without one, GPU power is left to the power domain. +pub(crate) fn has_pmp_link(pdev: &platform::Device) -> bool { + pdev.as_ref() + .of_node() + .is_some_and(|node| node.get_property::(c_str!("apple,pmp")).is_ok()) +} + +pub(crate) use crate::m3_firmware::{KnownImage, KNOWN_IMAGES, KNOWN_IMAGES_T8122}; +use crate::m3_firmware::image_info; + +fn sha256(data: &[u8]) -> [u8; 32] { + let mut digest = [0u8; 32]; + // SAFETY: `data` and `digest` are valid for the duration of the synchronous call. + unsafe { bindings::sha256(data.as_ptr(), data.len(), digest.as_mut_ptr()) }; + digest +} + +/// Lower-case hex of `bytes`, for logs. +fn hex(bytes: &[u8]) -> [u8; N] { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = [b'0'; N]; + for (i, b) in bytes.iter().take(N / 2).enumerate() { + out[2 * i] = DIGITS[(b >> 4) as usize]; + out[2 * i + 1] = DIGITS[(b & 0xf) as usize]; + } + out +} + +pub(crate) fn identify( + dev: &device::Device, + text: &[u8], + stkg_sha256: &[u8; 32], + images: &'static [KnownImage], +) -> Option<&'static KnownImage> { + let info = image_info(text); + let masked_sha256 = info.as_ref().and_then(|info| { + let mut copy = KVec::new(); + copy.extend_from_slice(text, GFP_KERNEL).ok()?; + copy.get_mut(info.patchbay.clone())?.fill(0); + Some(sha256(©)) + }); + let uuid = info.as_ref().map(|info| info.uuid); + let uuid_hex = hex::<32>(&uuid.unwrap_or([0; 16])); + let stkg_hex = hex::<64>(stkg_sha256); + let masked_hex = hex::<64>(&masked_sha256.unwrap_or([0; 32])); + fn as_str(b: &[u8]) -> &str { + core::str::from_utf8(b).unwrap_or("?") + } + dev_info!( + dev, + "M3: loaded GPU firmware: uuid {} (header {}), text sha256 {} (STKG zeroed), {} (patchbay zeroed)\n", + if uuid.is_some() { as_str(&uuid_hex) } else { "none" }, + if info.as_ref().is_some_and(|i| i.text_size == text.len()) { "ok" } else { "missing or unexpected" }, + as_str(&stkg_hex), + if masked_sha256.is_some() { as_str(&masked_hex) } else { "n/a" } + ); + let image = images.iter().find(|image| image.accepts_runtime(uuid, stkg_sha256)); + match image { + Some(image) => dev_info!(dev, "M3: GPU firmware identified as {}\n", image.name), + None => dev_err!(dev, "M3: loaded GPU firmware is not a known image\n"), + } + image +} + +/// Whether the reserved-memory node behind the GPU memory region `name` is no-map. +/// +/// A region without no-map is part of the kernel's linear map, which maps it write-back; any +/// other CPU mapping of it must use the same memory type, because mismatched aliases of the same +/// memory are not allowed. Only no-map regions may be mapped write-combined. Returns true when +/// the region cannot be looked up, which keeps the caller's own choice. +pub(crate) fn region_is_nomap(node: &of::Node, name: &CStr) -> bool { + if let Some(found) = static_region(node, name) { + return found.map_or(true, |(_, nomap)| nomap); + } + let Ok(names) = node.get_property::>(c_str!("memory-region-names")) else { + return true; + }; + let Some(index) = names.split(|b| *b == 0).position(|n| n == name.to_bytes()) else { + return true; + }; + let Some(region) = node.parse_phandle(c_str!("memory-region"), index) else { + return true; + }; + region.get_property::>(c_str!("no-map")).is_ok() +} + +/// Whether the M3 render node is registered (`asahi.m3_expose`). +/// +/// Userspace drivers pick up any registered render node. Until the M3 runtime is validated on a +/// board, a render node there would let a stock userspace driver submit work the board may not +/// run correctly. `auto` (-1) registers it only on the boards the runtime was validated on. +pub(crate) fn expose_render_node(soc: &Soc) -> bool { + match *crate::module_parameters::m3_expose.value() { + 0 => false, + v if v > 0 => true, + _ => runtime_validated_board(soc), + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs new file mode 100644 index 00000000000000..6661222530a38c --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Safety checks for clients of the M3 (G15) render node. +//! +//! Client admission: the M3 GPU runs AGX3 shaders and command streams. Userspace written for +//! the AGX2 GPUs (G13/G14) reads the M3's parameters as those of a G14 and would submit AGX2 +//! streams, which fault the GPU; the M3 backends cannot recover from a fault, so one such client +//! stops the GPU for the rest of the boot. Userspace that drives the M3 knows the AGX3 fields at +//! the end of `drm_asahi_params_global` (`usc_generation`, `gpu_hal_generation`) and asks +//! GET_PARAMS for the whole structure; older userspace asks for less. On G15 a file therefore +//! has to ask for the whole structure before it can create a VM. Every path to the GPU needs a +//! VM, so a refused client can still query the parameters but never reaches the hardware. + +use core::sync::atomic::{ + AtomicU32, + AtomicU64, + Ordering, // +}; + +use kernel::{ + prelude::*, + uapi, +}; + +use crate::{ + driver::AsahiDevice, + agx_uapi::{ + GpuAccess, + GpuAddressSpace, // + }, + mmu, +}; + +/// GET_PARAMS size an AGX3-aware client asks for: the whole global parameter structure. +const AGX3_PARAMS_SIZE: u64 = core::mem::size_of::() as u64; + +/// Refused clients that are logged per boot; later refusals only count. +const REFUSALS_LOGGED: u32 = 16; +static REFUSALS: AtomicU32 = AtomicU32::new(0); + +/// Per-file admission state. +pub(crate) struct ClientGate { + /// Largest size this file has asked GET_PARAMS for the global parameters with. + params_size: AtomicU64, +} + +impl ClientGate { + pub(crate) const fn new() -> Self { + Self { + params_size: AtomicU64::new(0), + } + } + + /// Record a successful GET_PARAMS for the global parameters. + pub(crate) fn note_params(&self, size: u64) { + self.params_size.fetch_max(size, Ordering::Relaxed); + } + + /// Whether this file may create a VM. Only G15 devices are gated. + pub(crate) fn admit_vm(&self, device: &AsahiDevice, flags: u32) -> Result { + let size = self.params_size.load(Ordering::Relaxed); + if crate::g16_profile::client_admitted(device.gpu()?.params()?.gpu_hal_generation, flags, size >= AGX3_PARAMS_SIZE) { + return Ok(()); + } + let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1); + if refusals <= REFUSALS_LOGGED { + let task = kernel::current!(); + let comm = task_comm(task); + let len = comm.iter().position(|&b| b == 0).unwrap_or(comm.len()); + let name = core::str::from_utf8(&comm[..len]).unwrap_or("?"); + let last = if refusals == REFUSALS_LOGGED { + " (further refusals are not logged)" + } else { + "" + }; + dev_info!( + device.as_ref(), + "M3: refusing a VM for {}[{}]: it asked GET_PARAMS for {} of the {} parameter bytes, so it cannot know this GPU runs AGX3 command streams; M3-aware userspace is needed{}\n", + name, + task.group_leader().pid(), + size, + AGX3_PARAMS_SIZE, + last + ); + } + Err(Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32))) + } +} + +/// The name of `task`, for log lines only. +pub(crate) fn task_comm(task: &kernel::task::Task) -> [u8; 16] { + let mut name = [0u8; 16]; + // SAFETY: `comm` is a fixed-size array inside the live task. It is read byte by byte with + // volatile loads because a concurrent rename may change it; a torn name is only printed. + unsafe { + let comm = core::ptr::addr_of!((*task.as_ptr()).comm).cast::(); + for (i, byte) in name.iter_mut().enumerate() { + *byte = comm.add(i).read_volatile(); + } + } + name +} + +/// Bytes at the start of a compute control stream whose launch records are checked. +const LAUNCH_WINDOW: usize = 4096; +/// Bytes that must be readable at a launch's program pointer. The pointer is 64-byte aligned, +/// so this never crosses a page. +const PROGRAM_BYTES: u64 = 64; +/// CDM record types (bits 31:29 of a record's first word). +const CDM_LAUNCH: u32 = 0; +const CDM_BARRIER: u32 = 3; + +/// Refuse a compute command whose control stream starts with a launch the GPU cannot fetch. +/// +/// Walks the records at the start of the stream: launches (40 bytes direct, 36 or 24 bytes +/// indirect, told apart by bits 28:27) and barriers (4 bytes). The walk ends at the first link, +/// return, terminator or unknown record, at the end of the stream and after `LAUNCH_WINDOW` +/// bytes. Each launch names its program at `(word1[31:22] << 38) | (word2 << 6)`; that address, +/// and the one read with only word2's low 26 bits when VA[37:32] is set, must be mapped +/// readable in the submitting VM, outside the driver's own mappings, or the command is refused +/// with EINVAL before it is queued. Without the check the GPU would take a translation fault on +/// the fetch, which the M3 backends cannot recover from. +/// +/// This catches malformed or stale pointers, not a hostile client: userspace can rewrite the +/// stream after the check. +pub(crate) fn check_compute_launches( + vm: &mmu::Vm, + space: &A, + base: u64, + end: u64, +) -> Result { + let len = usize::try_from(end.checked_sub(base).ok_or(EINVAL)?) + .unwrap_or(usize::MAX) + .min(LAUNCH_WINDOW); + let mut stream = KVec::from_elem(0u8, len, GFP_KERNEL)?; + if let Err(error) = read_through_cache(vm, base, &mut stream) { + pr_err!( + "M3 compute rejected: control stream {:#x} cannot be read ({:?})\n", + base, + error + ); + return Err(EINVAL); + } + let word = |offset: usize| { + u32::from_le_bytes([ + stream[offset], + stream[offset + 1], + stream[offset + 2], + stream[offset + 3], + ]) + }; + let mut offset = 0; + let mut launch = 0u32; + while offset + 4 <= len { + let head = word(offset); + match head >> 29 { + CDM_LAUNCH => { + if offset + 12 > len { + break; + } + let high = u64::from(word(offset + 4) >> 22) << 38; + let low = u64::from(word(offset + 8)); + // Whether the GPU takes VA[37:32] from word2's top six bits is not + // established, so the program must be mapped under both readings. They + // agree whenever those bits are clear, as in every working launch. + for program in [high | (low << 6), high | ((low & 0x3ff_ffff) << 6)] { + if !space.covers(program, PROGRAM_BYTES, GpuAccess::Read) { + pr_err!( + "M3 compute rejected: launch {} at {:#x} names program {:#x}, which is not mapped readable in this VM\n", + launch, + base + offset as u64, + program + ); + return Err(EINVAL); + } + } + launch += 1; + offset += match (head >> 27) & 3 { + 0 => 40, + 1 => 36, + 2 => 24, + _ => break, + }; + } + CDM_BARRIER => offset += 4, + _ => break, + } + } + Ok(()) +} + +/// Read `out.len()` bytes at `iova` in `vm` through the kernel's linear map. +/// +/// Userspace normally writes command streams through a write-combined mapping, which does not +/// update cache lines the kernel's cacheable alias may hold from an earlier read or a +/// speculative fetch. Each line is cleaned and invalidated before it is read, so the bytes +/// come from memory, as the GPU will fetch them. Cleaning never discards data: a dirty line from +/// a cached mapping is written back first. +/// +/// `Vm::read_mapped_bytes` holds the VM execution lock from translation until the copy, so the +/// pages cannot be unmapped and freed while they are cleaned and read. +fn read_through_cache(vm: &mmu::Vm, iova: u64, out: &mut [u8]) -> Result { + vm.read_mapped_bytes(iova, out, |src, len| clean_invalidate(src as usize, len)) +} + +/// Clean and invalidate the data cache lines covering `start..start + len` to the point of +/// coherency, completing before later loads. +fn clean_invalidate(start: usize, len: usize) { + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1 and has no side effects. + unsafe { + core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, options(nomem, nostack, preserves_flags)); + } + // DminLine: log2 of the smallest data cache line, in 4-byte words. + let line = 4usize << ((ctr >> 16) & 0xf); + let end = start.saturating_add(len); + let mut address = start & !(line - 1); + // SAFETY: the range lies inside one page of the linear map that the caller has mapped. + // Clean-and-invalidate writes dirty lines back before dropping them, so no data is lost. + unsafe { + core::arch::asm!("dsb sy", options(nostack, preserves_flags)); + while address < end { + core::arch::asm!("dc civac, {address}", address = in(reg) address, options(nostack, preserves_flags)); + address += line; + } + core::arch::asm!("dsb sy", options(nostack, preserves_flags)); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs new file mode 100644 index 00000000000000..08114c4d8cc8ad --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Completion work owned only by the M3 runtime. + +use kernel::{prelude::*, sync::Arc, workqueue::{self, impl_has_work, new_work, Work, WorkItem}}; +use crate::{driver, m3_drm::Shared}; + +#[pin_data] +pub(crate) struct Completion { + shared: Shared, + #[pin] + work: Work, +} +impl Completion { + pub(crate) fn new(shared: Shared) -> Result> { + Arc::pin_init(try_pin_init!(Self { + shared, + work <- new_work!("M3 completion"), + }), GFP_KERNEL) + } +} +impl_has_work! { impl HasWork for Completion { self.work } } +impl WorkItem for Completion { + type Pointer = Arc; + fn run(owner: Arc) { + use crate::m3_runtime::Service; + // Snapshots after a notification before sleeping for the next one: the completion + // stamps can trail the firmware's event message. + const TRAILING_POLLS: u32 = 4; + let (mut trailing, mut waited_at) = (0, u64::MAX); + loop { + let (outcome, events) = { + let mut guard = owner.shared.lock(); + let Some(runtime) = Option::as_mut(&mut *guard) else { return; }; + let outcome=runtime.service_job(); + let events=runtime.events(); + // Preserve the root-only CPU-overlap diagnostic: disabled waits retain + // the runtime lock so VM/timestamp preparation cannot overlap the GPU. + if let Service::Waiting(messages)=outcome { + if !events.cpu_overlap() {events.wait_past(messages);continue;} + } + (outcome,events) + }; + match outcome { + Service::Idle => break, + Service::Progress => { trailing = 0; waited_at = u64::MAX; } + Service::Waiting(messages) => { + if messages != waited_at { waited_at = messages; trailing = 0; } + if trailing < TRAILING_POLLS { + trailing += 1; + kernel::time::delay::fsleep(kernel::time::Delta::from_micros(5)); + } else { + // Until the next firmware event, or a bounded 100 us resnapshot. + events.wait_past(messages); + trailing = 0; + } + } + } + } + } +} +pub(crate) fn queue(dev: &driver::AsahiDevice) { + if let Some(work) = dev.completion.as_ref() { + let _ = workqueue::system_highpri().enqueue(work.clone()); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs new file mode 100644 index 00000000000000..e7469d31b496fe --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned RTKit 2419 M3 compute graph, populated with a validated user CDM. +use kernel::prelude::*; +use crate::m3_queue_layout as queue; +use crate::m3_state_layout as state; +use crate::{m3_compute_layout as command, m3_compute_sequence as microsequence, + m3_pool_layout::GpuRegion, m3_init_layout::Region}; +use crate::{driver,mmu,agx_memory::{self},m3_memory::Buffer,m3_compute_storage as storage}; +use crate::m3_timeline as timeline; +use storage::{NOTIFIER,PREEMPTION,GPU_CONTEXT,JOB_LIST,QUEUE_SCRATCH,TIMESTAMPS, + TAIL_SCRATCH,MICROSEQUENCE_SCRATCH,Mapping}; +const UMA_POOL:crate::m3_pool::Owners=crate::m3_pool::Owners { + backing:storage::POOL_BACKING,pages:storage::POOL_PAGES,table:storage::POOL_TABLE, + manager:storage::POOL,counter:storage::COUNTER,table_index:2, +}; +// Retain at most four VM views, including the active one, as the render +// graph does. Game and compositor compute jobs otherwise remap the whole +// private arena every time they alternate. Each entry pins its VM binding. +struct CachedViews { + command: mmu::KernelMapping, + views: KVec<(usize, mmu::KernelMapping)>, + // Release cached mappings before returning their ASID lease. + binding: mmu::VmBind, +} +pub(crate) struct Compute { + _client_command: mmu::KernelMapping, + objects: KVec, + command_bytes: KVec, + sequence_bytes: KVec, + _binding: mmu::VmBind, + stats: Region, + sequence: u64, + last_progress_ns: i64, + head: u16, + cached_views: KVec, + slot: usize, + batch_count: usize, + checkpoint: (u64,u16,usize), + /// Firmware-only barrier storage, allocated only for mixed publications. + barriers: Option, + /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). + registers: command::RegisterSet, +} +impl Compute { + pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, + stats: Region, control: crate::agx_compute::Control, + registers: command::RegisterSet) -> Result { + let binding=uat.bind(vm)?; + let mut objects=KVec::new(); + let mut client_command=None; + let mut command_bytes = KVec::new(); + command_bytes.resize(command::SIZE, 0u8, GFP_KERNEL)?; + let mut sequence_bytes = KVec::new(); + sequence_bytes.resize(microsequence::STORAGE, 0u8, GFP_KERNEL)?; + // A private compute aperture, disjoint from the retained render graph. + // Address-bearing graph fields are constructed from their live owners. + for index in 0..storage::COUNT { + let a=storage::batch_allocation(index).map_err(|_|EINVAL)?; + let mut b=match a.mapping { + Mapping::CommandAlias(va)=>Buffer::at(dev,uat.kernel_vm(),Some((uat.kernel_lower_vm(),va)),None,a.size,true)?, + Mapping::Client(va)=>Buffer::at(dev,uat.kernel_vm(),Some((vm,va)),None,a.size,true)?, + Mapping::FirmwareGpuShared=>Buffer::at(dev,uat.kernel_vm(),None,None,a.size,true)?, + Mapping::FirmwareOnly=>Buffer::at(dev,uat.kernel_vm(),None,None,a.size,false)?, + }; + if matches!(a.mapping,Mapping::CommandAlias(_)) {client_command=Some(b.map_gpu_view(vm)?);} + objects.push(b,GFP_KERNEL)?; + } + Self::encode_command(&mut objects, &mut command_bytes, binding.slot(), control, 1, 0, registers)?; + Self::encode_sequence(&mut objects, &mut sequence_bytes, stats, binding.slot(), timeline::stamp(0xc1000000,1), 0)?; + UMA_POOL.initialize(&mut objects)?; + Self::initialize_state(&mut objects, binding.slot())?; + // Queue references are firmware VAs from live allocations. In contrast + // to render's unpublished seed entries, compute dispatches entry zero + // immediately, so its initial producer cursor remains one. + let address = |index: usize| queue::FirmwareVa::new(objects[index].va()).map_err(|_| EINVAL); + let info = queue::QueueInfo { state: address(storage::QUEUE_STATE)?, ring: address(storage::RING)?, + job_list: address(JOB_LIST)?, gpu_scratch: address(QUEUE_SCRATCH)?, + context: address(GPU_CONTEXT)?, uuid: 0xc10000 }; + let command = address(storage::COMMAND)?; + let mut bytes = [0; queue::INFO_SIZE]; + info.encode(&mut bytes).map_err(|_| EINVAL)?; + objects[storage::QUEUE].write(0, &bytes)?; + let mut state = [0; queue::STATE_SIZE]; + queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 1 } + .encode(&mut state).map_err(|_| EINVAL)?; + objects[storage::QUEUE_STATE].write(0, &state)?; + for index in 0..queue::QUALIFIED_CAPACITY { + let entry = queue::ring_entry(index, queue::QUALIFIED_CAPACITY, &[command]).map_err(|_| EINVAL)?; + objects[storage::RING].write(index as usize * 8, &entry)?; + } + dev_info!(dev.as_ref(),"M3 compute graph: context={} queue={:#x} command={:#x} low={:#x} cdm={:#x}..{:#x} pool={:#x}\n", + binding.slot(),objects[storage::QUEUE].va(),objects[storage::COMMAND].va(), + objects[storage::COMMAND].gpu_va()?,control.base,control.end,objects[storage::POOL].va()); + // The M3 compute stream uses absolute IOTO addresses and does not + // include the USC_EXEC_BASE_CP register. + let mut stream=[0u8;48]; + vm.read_bytes(control.base,&mut stream)?; + crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",stream); + agx_memory::publish(); + // Finish the write-combining page-table stores before broadcasting + // the invalidation. + crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers}) + } + fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, + control: crate::agx_compute::Control, sequence: u64, slot: usize, + register_set: command::RegisterSet) -> Result { + let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); + let preemption = &objects[PREEMPTION]; + let value = command::Command { + context, counter: timeline::previous(sequence), notifier: fw(NOTIFIER)?, + preemption: GpuRegion::new(preemption.gpu_va()?+offset(PREEMPTION)? as u64, storage::allocation(PREEMPTION).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL)?, + cdm: command::GpuVa::new(control.base).map_err(|_| EINVAL)?, + cdm_last: command::GpuVa::new(control.end.checked_sub(4).ok_or(EINVAL)?).map_err(|_| EINVAL)?, + gpu_alias: command::GpuVa::new(objects[storage::COMMAND].gpu_va()?+offset(storage::COMMAND)? as u64).map_err(|_| EINVAL)?, + microsequence: fw(storage::MICROSEQUENCE)?, microsequence_size: microsequence::LENGTH as u32, + stamp: fw(storage::STAMP)?, fw_stamp: fw(storage::FW_STAMP)?, + stamp_value: timeline::stamp(0xc1000000,sequence), + timestamps: [fw(TIMESTAMPS[0])?, fw(TIMESTAMPS[1])?], + pool: fw(storage::POOL)?, tail_scratch: fw(TAIL_SCRATCH)?, flush_stamps: false, + register_set, + }; + value.encode(bytes).map_err(|_| EINVAL)?; + objects[storage::COMMAND].write(offset(storage::COMMAND)?, bytes) + } + fn encode_sequence(objects: &mut [Buffer], bytes: &mut [u8], stats: Region, + context: u32, stamp_value: u32, slot: usize) -> Result { + let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); + let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); + let region = |owner: usize| Region::new(objects[owner].va()+offset(owner)? as u64, storage::allocation(owner).map_err(|_|EINVAL)?.size).map_err(|_| EINVAL); + let value = microsequence::Sequence { + command: region(storage::COMMAND)?, stats, notifier: region(NOTIFIER)?, + queue: fw(storage::QUEUE)?, context, generation: 1, pool: fw(storage::POOL)?, + scratch: [fw(MICROSEQUENCE_SCRATCH[0])?, fw(MICROSEQUENCE_SCRATCH[1])?, + fw(MICROSEQUENCE_SCRATCH[2])?, fw(MICROSEQUENCE_SCRATCH[3])?, fw(MICROSEQUENCE_SCRATCH[4])?], + fw_stamp: fw(storage::FW_STAMP)?, stamp_value, + }; + value.encode(bytes).map_err(|_| EINVAL)?; + objects[storage::MICROSEQUENCE].write(offset(storage::MICROSEQUENCE)?, bytes) + } + fn initialize_state(objects: &mut [Buffer], context_slot: u32) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(objects[owner].va()).map_err(|_| EINVAL); + let notification = state::Notifier { threshold: address(storage::EVENT)?, context: context_slot }; + let list_owner = address(JOB_LIST)?; + let mut context = [0; state::CONTEXT_SIZE]; + state::context(&mut context).map_err(|_| EINVAL)?; + objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + objects[NOTIFIER].write(0, ¬ifier)?; + let mut list = [0; state::JOB_LIST_SIZE]; + state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; + objects[JOB_LIST].write(0, &list)?; + for (owner, value) in [(storage::COUNTER, 1), (storage::EVENT, 1), (storage::STAMP, timeline::stamp(0xc1000000,0))] { + let mut bytes = [0; state::MANAGER_STATE_SIZE]; + let bytes = bytes.get_mut(..objects[owner].size()).ok_or(EINVAL)?; + state::counter(bytes, value).map_err(|_| EINVAL)?; + objects[owner].write(0, bytes)?; + } + // Fill the preemption buffer with 0xcc, including when a compute + // command is submitted again. + objects[PREEMPTION].fill(0xcc) + } + pub(crate) fn replay(&mut self, uat: &mmu::Uat, vm: &mmu::Vm, + control: crate::agx_compute::Control) -> Result { + if !self.complete()? { return Err(EBUSY); } + // All prior flushed stamps/events and engine idle were checked by + // Runtime before entry. Keep pool/queue ownership, remap only client + // views and patch the context when switching Vulkan processes. + if !self._binding.matches(vm) { + let next = if let Some(index) = self.cached_views.iter().position(|v| v.binding.matches(vm)) { + self.cached_views.remove(index).map_err(|_| EIO)? + } else { + let binding = uat.bind(vm)?; + let mut views = KVec::new(); + for i in 0..storage::COUNT { + if matches!(storage::allocation(i).map_err(|_|EINVAL)?.mapping,Mapping::Client(_)) { + views.push((i,self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; + } + } + let command = self.objects[storage::COMMAND].map_gpu_view(vm)?; + CachedViews {binding, command, views} + }; + // Complete allocations before moving the active views. Eviction + // drops only an idle cached view; no arena storage is duplicated. + let old_views = KVec::with_capacity(next.views.len(), GFP_KERNEL)?; + self.cached_views.reserve(1, GFP_KERNEL)?; + if self.cached_views.len() == 3 { + drop(self.cached_views.remove(0).map_err(|_| EIO)?); + } + let mut old = CachedViews { + binding: core::mem::replace(&mut self._binding, next.binding), + command: core::mem::replace(&mut self._client_command, next.command), + views: old_views, + }; + for (i, view) in next.views { + old.views.push((i, self.objects[i].swap_gpu_view(view)?), GFP_KERNEL)?; + } + self.cached_views.push(old, GFP_KERNEL)?; + } + self.checkpoint=(self.sequence,self.head,self.batch_count); + self.batch_count=0; + if let Err(error)=self.append(control) { + self.abort_batch()?; + return Err(error); + } + Ok(()) + } + /// Append only while the previous batch is fully retired and this one + /// remains unpublished. Firmware serializes each command, including its + /// original completion flush, in this one compute queue. + pub(crate) fn append(&mut self,control:crate::agx_compute::Control)->Result { + if self.batch_count>=storage::SLOTS {return Err(E2BIG);} + self.slot=self.batch_count; + for i in storage::RESET { + let offset=storage::slot_offset(i,self.slot).map_err(|_|EINVAL)?; + let size=storage::allocation(i).map_err(|_|EINVAL)?.size; + self.objects[i].fill_range(offset,size,if i==PREEMPTION {0xcc} else {0})?; + } + self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT,self._binding.slot())?; + self.sequence=self.sequence.checked_add(1).ok_or(EOVERFLOW)?; + Self::encode_command(&mut self.objects,&mut self.command_bytes,self._binding.slot(),control,self.sequence,self.slot,self.registers)?; + let stamp=self.stamp(); + Self::encode_sequence(&mut self.objects,&mut self.sequence_bytes,self.stats,self._binding.slot(),stamp,self.slot)?; + self.objects[storage::EVENT].u32(0,self.sequence as u32)?; + self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + let address=self.objects[storage::COMMAND].va()+storage::slot_offset(storage::COMMAND,self.slot).map_err(|_|EINVAL)? as u64; + self.objects[storage::RING].u64(usize::from(self.head)*8,address)?; + self.head=(self.head+1)%queue::QUALIFIED_CAPACITY as u16; + self.batch_count+=1; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + Ok(()) + } + /// Roll back producer state before any SubmitQueue publication. Old ring + /// entries beyond the restored cursor are unreachable and get overwritten. + pub(crate) fn abort_batch(&mut self)->Result { + (self.sequence,self.head,self.batch_count)=self.checkpoint; + self.slot=self.batch_count.saturating_sub(1); + self.objects[storage::EVENT].u32(0,self.sequence as u32)?; + self.objects[storage::COUNTER].u32(0,self.sequence as u32)?; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();Ok(()) + } + fn offset(&self,owner:usize)->Result { + storage::slot_offset(owner,self.slot).map_err(|_|EINVAL) + } + /// Same cache-generation contract as M4: StartCompute and the persistent + /// notifier must use the same tag before the firmware flushes user stamps. + /// RTKit 2419: StartCompute.counter1 +0x34; JobMeta.flush_stamps +0x804. + pub(crate) fn prepare_completion(&mut self,coalesce:bool)->Result { + let sequence=self.offset(storage::MICROSEQUENCE)?; + let command=self.offset(storage::COMMAND)?; + self.objects[storage::MICROSEQUENCE].u32(sequence+microsequence::GENERATION,0)?; + self.objects[storage::COMMAND].u32(command+command::FLUSH_STAMPS,u32::from(!coalesce))?; + agx_memory::publish();Ok(()) + } + pub(crate) fn set_user_timestamps(&mut self,addresses:[u64;2])->Result { + let offset=self.offset(storage::COMMAND)?+command::USER_TIMESTAMPS; + self.objects[storage::COMMAND].u64(offset,addresses[0])?; + self.objects[storage::COMMAND].u64(offset+8,addresses[1])?; + let pointer=if addresses==[0,0] {0} else {self.objects[storage::COMMAND].va()+offset as u64}; + let sequence=self.offset(storage::MICROSEQUENCE)?; + for start in [microsequence::TIMESTAMP_START,microsequence::TIMESTAMP_END] { + self.objects[storage::MICROSEQUENCE].u64(sequence+start+microsequence::USER_TIMESTAMP_POINTER,pointer)?; + } + agx_memory::publish();Ok(()) + } + pub(crate) fn set_attachments(&mut self,attachments:&crate::agx_attachments::Attachments)->Result { + let sequence=self.offset(storage::MICROSEQUENCE)?; + self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; + self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; + agx_memory::publish();Ok(()) + } + fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } + pub(crate) fn completion_dependency(&self)->Result { + Ok(crate::m3_sync_layout::CompletionDependency { + stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?, + value:self.stamp(),event:2, + }) + } + pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat)->Result { + if self.barriers.is_none() { + self.barriers=Some(Buffer::at(dev,uat.kernel_vm(),None,None,storage::SLOTS*64,false)?); + } + Ok(()) + } + /// Insert a dependency before the last appended, still unpublished CDM. + /// Its command ordinal and timestamp slot are unchanged; only the ring + /// gains an entry. The whole batch must retire before this storage is reused. + pub(crate) fn insert_dependency(&mut self,dependency:crate::m3_sync_layout::CompletionDependency)->Result { + if self.batch_count==0 || self.slot>=storage::SLOTS || dependency.event!=1 { + return Err(EINVAL); + } + let value=self.stamp(); + let barriers=self.barriers.as_mut().ok_or(EINVAL)?; + let offset=self.slot*64; + let mut bytes=[0;crate::m3_sync_layout::BARRIER_SIZE]; + crate::m3_sync_layout::Barrier { + stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, + self_value:value,uuid:0xc10000,dependency:crate::m3_sync_layout::Dependency::PreviousPass, + }.encode(&mut bytes).map_err(|_|EINVAL)?; + barriers.write(offset,&bytes)?; + let address=barriers.va()+offset as u64; + let (slots,next)=queue::insert_before_last(self.head,queue::QUALIFIED_CAPACITY).map_err(|_|EINVAL)?; + let command=self.objects[storage::COMMAND].va()+self.offset(storage::COMMAND)? as u64; + self.objects[storage::RING].u64(usize::from(slots[0])*8,address)?; + self.objects[storage::RING].u64(usize::from(slots[1])*8,command)?; + self.head=next; + self.objects[storage::QUEUE_STATE].u32(queue::WRITE,u32::from(self.head))?; + agx_memory::publish();crate::mem::sync();Ok(()) + } + pub(crate) fn gpu_ns(&mut self)->Result { + let mut total=0; + for slot in 0..self.batch_count {total+=self.slot_gpu_ns(slot)?;} + Ok(total) + } + pub(crate) fn slot_gpu_ns(&mut self,slot:usize)->Result { + if slot>=self.batch_count {return Err(EINVAL);} + let start=self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],slot).map_err(|_|EINVAL)?)?; + let end=self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],slot).map_err(|_|EINVAL)?)?; + Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + } + /// A batch span includes gaps between its compute commands, unlike gpu_ns. + /// Read only after stamps, queues, events and idle state prove retirement. + pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { + if self.batch_count == 0 {return Err(EINVAL);} + Ok([self.objects[TIMESTAMPS[0]].read_u64(storage::slot_offset(TIMESTAMPS[0],0).map_err(|_|EINVAL)?)?, + self.objects[TIMESTAMPS[1]].read_u64(storage::slot_offset(TIMESTAMPS[1],self.batch_count-1).map_err(|_|EINVAL)?)?]) + } + pub(crate) fn head(&self) -> u16 { self.head } + pub(crate) fn ordinal(&self)->u64 {self.sequence} + pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } + pub(crate) fn progress(&mut self, dev: &driver::AsahiDevice) -> Result { + // Report real completions by elapsed time as well as sequence count; + // batches need not land on a multiple of 128. + if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + let now = ::ktime_get(); + if self.last_progress_ns != 0 && now - self.last_progress_ns < 1_000_000_000 + && self.sequence != 1 && self.sequence % 128 != 0 + && !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + self.last_progress_ns = now; + dev_info!(dev.as_ref(), "M3_COMPUTE_RETIRED sequence={} context={} stamp={:#x}/{:#x} queue={}/{} head={} idle=1 cache_flush=1\n", + self.sequence, self._binding.slot(), self.objects[storage::STAMP].read_u32(0)?, + self.objects[storage::FW_STAMP].read_u32(0)?, self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?, + self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?, self.head); + Ok(()) + } + + pub(crate) fn queue(&self)->u64 {self.objects[storage::QUEUE].va()} + pub(crate) fn complete(&mut self)->Result { + Ok(self.objects[storage::STAMP].read_u32(0)?==self.stamp() + && self.objects[storage::FW_STAMP].read_u32(0)?==self.stamp() + && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) + && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) + } + pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + for off in (0..512).step_by(128) { + let mut raw=[0u8;128]; self.objects[storage::COMMAND].read(off,&mut raw)?; + dev_info!(dev.as_ref(),"M3 post-command {:#x} {:02x?}\n",off,raw); + } + let stamp=self.objects[storage::STAMP].read_u32(0)?; + let fw=self.objects[storage::FW_STAMP].read_u32(0)?; + let done=self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?; + let read=self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?; + dev_info!(dev.as_ref(),"M3 compute: stamps={:#x}/{:#x} done={} read={}\n",stamp,fw,done,read); + let mut record=[0u8;0x84];self.objects[storage::POOL].read(0,&mut record)?; + dev_info!(dev.as_ref(),"M3 compute pool={:02x?}\n",record); + let mut roots=[(0,0);64];self._binding.vm().context_roots(&mut roots)?; + dev_info!(dev.as_ref(),"M3 context {} roots={:x?}, bootstrap={:x?}\n",self._binding.slot(),roots[self._binding.slot() as usize],roots[0]); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs new file mode 100644 index 00000000000000..59277aaefb6817 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned RTKit 2419.140.12 configuration with named initialization references. +use kernel::prelude::*; +use crate::m3_init_layout as init; +use crate::m3_init_storage as storage; +use storage::{REGION_A,RUNTIME_POINTERS,HARDWARE_DATA,UNKNOWN_PAIR,FWLOG_PAYLOAD, + UNKNOWN_C0,UNKNOWN_C1,UNKNOWN_C3,GLOBALS,GLOBALS_POWER,CONTROL_REGION,RUNTIME_FLAGS, + FW_CONTROL_STATE,FW_CONTROL_RING}; +use crate::{driver,agx_memory::{self},m3_memory::Buffer,mmu,pgtable::prot}; +kernel::static_assert!(core::mem::size_of::() == init::fwlog::ENTRY_SIZE); +kernel::static_assert!(core::mem::size_of::() == init::fwlog::PAYLOAD_SIZE); +kernel::static_assert!(::SUB_CHANNELS == init::fwlog::CHANNELS); + +/// The root, runtime-pointer and firmware-control records, encoded for the owners +/// `region` describes. Config uploads them for its own allocations; the device-tree +/// InitData checks encode them for the fixed allocation table before GPU access. +pub(crate) struct Records { + pub(crate) root: [u8; init::ROOT_SIZE], + pub(crate) runtime: [u8; init::RUNTIME_SIZE], + pub(crate) control: KVec, +} +impl Records { + pub(crate) fn encode(region: impl Fn(usize) -> Result) -> Result { + let root = init::Root { region_a: region(REGION_A)?, runtime: region(RUNTIME_POINTERS)?, + globals: region(GLOBALS)?, new_a: region(RUNTIME_FLAGS)?, + control: region(CONTROL_REGION)?, power: region(GLOBALS_POWER)? }; + let mut channels = [init::Channel { state: region(0)?, ring: region(1)? }; 17]; + for (index, channel) in channels.iter_mut().enumerate() { + *channel = init::Channel { state: region(index * 2)?, ring: region(index * 2 + 1)? }; + } + let runtime = init::RuntimePointers { hardware: region(HARDWARE_DATA)?, + unknown_pair: region(UNKNOWN_PAIR)?, fwlog_payload: region(FWLOG_PAYLOAD)?, + unknown_c0: region(UNKNOWN_C0)?, unknown_c1: region(UNKNOWN_C1)?, + unknown_c3: region(UNKNOWN_C3)?, channels }; + let control = init::ControlRegion { channel: init::Channel { + state: region(FW_CONTROL_STATE)?, ring: region(FW_CONTROL_RING)? } }; + let mut root_bytes = [0; init::ROOT_SIZE]; + root.encode(&mut root_bytes).map_err(|_| EINVAL)?; + let mut runtime_bytes = [0; init::RUNTIME_SIZE]; + runtime.encode(&mut runtime_bytes).map_err(|_| EINVAL)?; + // The control/status region is 50 KiB; use temporary heap storage, + // never a kernel-stack array. All three owners precede firmware startup. + let mut control_bytes = KVec::new(); + control_bytes.resize(init::CONTROL_SIZE, 0u8, GFP_KERNEL)?; + control.encode(&mut control_bytes).map_err(|_| EINVAL)?; + Ok(Self { root: root_bytes, runtime: runtime_bytes, control: control_bytes }) + } +} + +pub(crate) struct Config { + objects: KVec, + #[cfg(CONFIG_DEV_COREDUMP)] + fault_reserve: Option, + _iomaps: KVec, + pub(crate) completed_events: u64, + pstates: crate::m3_adt_config::PstateWatch, + thermal: crate::m3_thermal::Governor, + request_top: u32, + request_value: u32, + request_checked: bool, + request_check_at: Option>, + request_writes: u64, + request_observed: u32, +} +impl Config { + /// Snapshot owned RAM only. Firmware can still update these buffers; + /// per-record time intervals explicitly describe sequential observations. + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn fault_snapshot(&mut self, dev: &driver::AsahiDevice, primary: Error, gpu_pending: bool) -> Result { + let mut dump = self.fault_reserve.take().ok_or(ENOMEM)?; + dump.begin(0); + let mut cause = [0u8; 16]; + cause[..4].copy_from_slice(&1u32.to_le_bytes()); + cause[4..8].copy_from_slice(&primary.to_errno().to_le_bytes()); + cause[8..16].copy_from_slice(&u64::from(gpu_pending).to_le_bytes()); + dump.record("host-first-error", 0, cause.len(), |out| { + out.copy_from_slice(&cause); Ok(()) + })?; + for (name, index) in [ + ("root", storage::ROOT), ("runtime", RUNTIME_POINTERS), + ("hardware", HARDWARE_DATA), ("globals", GLOBALS), + ("control", CONTROL_REGION), ("flags", RUNTIME_FLAGS), + // The power-controller block, and the host-to-firmware device-control and + // firmware-control queues: their state words show whether the firmware read a + // queued control (read index) and what it was sent. + ("power", GLOBALS_POWER), + ("devctl-state", storage::DEVICE_CONTROL * 2), + ("devctl-ring", storage::DEVICE_CONTROL * 2 + 1), + ("fwctl-state", storage::FW_CONTROL_STATE), + ("fwctl-ring", storage::FW_CONTROL_RING), + ("event-state", storage::EVENT * 2), + ("event-ring", storage::EVENT * 2 + 1), + ("fwlog-state", storage::FIRMWARE_LOG * 2), + ("fwlog-ring", storage::FIRMWARE_LOG * 2 + 1), + ("fwlog-payload", FWLOG_PAYLOAD), + ("trace-state", storage::TRACE * 2), + ("trace-ring", storage::TRACE * 2 + 1), + ] { + let buffer = &mut self.objects[index]; + dump.record(name, buffer.va(), buffer.size(), |out| buffer.read(0, out))?; + } + dump.publish(dev.as_ref()) + } + + pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, firmware: &crate::m3_firmware::Firmware, + contents: &crate::m3_adt_config::Contents) -> Result { + contents.check_for_upload()?; + let mut objects = KVec::new(); + let mut iomaps = KVec::new(); + for index in 0..storage::COUNT { + let mut a=storage::allocation(index).map_err(|_|EINVAL)?; + let image=contents.initial(index,&a)?; + if a.initial!=storage::Initial::Zero && image.len()!=a.size {return Err(EINVAL);} + // The SoC may place the HwData object elsewhere and have it mapped cacheable + // (`Soc::hwdata_object`); the records and the upload check use the live address. + let object=if index==HARDWARE_DATA {contents.hwdata_object} else {None}; + if let Some(o)=object {a.address=o.address;} + let mut b=match object { + Some(o) if o.cached=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address),a.size, + prot::PROT_FW_PRIV_RW,prot::PROT_GPU_SHARED_RW)?, + _=>Buffer::at(dev,uat.kernel_vm(),None,Some(a.address),a.size,a.gpu_shared)?, + }; + b.write(0,image)?;objects.push(b,GFP_KERNEL)?; + } + for io in contents.iomaps { + // Read-write unless the contents say otherwise (only the T8122 start experiment). + let protection=if contents.read_only_slots==0 {prot::PROT_FW_MMIO_RW} + else {crate::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; + let map=uat.kernel_vm().map_io(io.address,io.physical.try_into()?,io.size,protection)?; + let owned=init::Region::new(map.iova(),io.size).map_err(|_|EINVAL)?; + let field=io.pointer_field().map_err(|_|EINVAL)?; + objects[HARDWARE_DATA].u64(field,io.pointer(owned).map_err(|_|EINVAL)?)?; + iomaps.push(map,GFP_KERNEL)?; + } + // This field is a physical address owned by the reserved GPU region, + // unlike the firmware virtual references in the IOMapping records. + objects[HARDWARE_DATA].u64(storage::GPU_REGION_PHYSICAL,firmware.resources.regions[0].base)?; + Self::initialize_records(&mut objects)?; + // The root record carries the G15 14.8.3 InitData version, which is every T6030 + // image's. Only the T8122 start experiment gives the firmware another one + // (`asahi.t8122_initdata_version`). + if firmware.initdata_magic!=crate::m3_firmware::G15_V14_8_3_INITDATA { + objects[storage::ROOT].u64(0,firmware.initdata_magic)?; + } + // Vertex/tessellation loops need not retire a primitive at every + // firmware progress poll. The inherited interval of 10 falsely + // declares a valid 35.7 ms TA job stuck on J514S. Keep progress + // detection enabled, with a finite interval of 100 firmware polls. + // This is independent of the host job bound and 60-second watchdog. + // RTKit 2419 copies Globals+0x980 into its TA poll threshold at + // text+0x6514; text+0xd710 checks it before sampling engine progress. + const TA_PROGRESS_INTERVAL: usize = 0x980; + if objects[GLOBALS].read_u32(TA_PROGRESS_INTERVAL)? != 10 { return Err(EINVAL); } + objects[GLOBALS].u32(TA_PROGRESS_INTERVAL, 100)?; + dev_info!(dev.as_ref(), "M3: firmware TA progress-check interval=100\n"); + // RTKit 2419 +2de48..2dec8 accepts user timestamp stores only + // within the 64 MiB arena at HwDataB+28. Match map_timestamp(). + objects[HARDWARE_DATA].u64(0x28,agx_memory::TIMESTAMP_RANGE.start)?; + // RTKit 2419 HwDataA follows the 0x4580-byte HwDataB. Request the + // boot performance state through firmware (asahi.m3_boot_pstate) and cap + // the states it may use (asahi.m3_max_pstate), retaining the voltage + // tables, power/thermal ceilings and minimum operating state. + // The bring-up image initialized actual/target and every base to 1 + // (338 MHz), so short desktop jobs never left that startup state. + // The uploaded table is the SoC's (`Soc::pstates`): its highest state at +0xb58 and that + // state's frequency in the table at +0xb5c (8 and 1380 MHz on T6030). + let (states, top_mhz) = contents.table; + if objects[HARDWARE_DATA].read_u32(0xb58)? != states + || objects[HARDWARE_DATA].read_u32(0xb5c + states as usize * 4)? != top_mhz + { + return Err(EINVAL); + } + let (hwdata, globals) = objects.split_at_mut(GLOBALS); + crate::m3_adt_config::publish_pstates(dev.as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; + // Firmware ktrace is for explicit detailed diagnostics. Normal games + // still drain error/events, but need not generate trace records for + // every firmware operation on the command-processing core. + objects[RUNTIME_FLAGS].u32(0,u32::from(crate::debug::debug_enabled( + crate::debug::DebugFlags::SubmitTiming)))?; + crate::m3_adt_config::check_upload(dev.as_ref(), firmware, contents.iomaps, &mut objects)?; + agx_memory::publish(); + let thermal = crate::m3_thermal::Governor::new(dev.as_ref(), &contents.pstates)?; + #[cfg(CONFIG_DEV_COREDUMP)] + let fault_reserve = match crate::agx_fault::Dump::new_m3() { + Ok(dump) => Some(dump), + Err(error) => { + dev_warn!(dev.as_ref(), "M3: fault snapshot reserve unavailable: {:?}\n", error); + None + } + }; + Ok(Self { objects, _iomaps: iomaps, completed_events: 0, + #[cfg(CONFIG_DEV_COREDUMP)] fault_reserve, + pstates: crate::m3_adt_config::PstateWatch::new(contents.pstates), thermal, + request_top:contents.table.0,request_value:0,request_checked:false, + request_check_at:None,request_writes:0,request_observed:0 }) + } + fn initialize_records(objects: &mut [Buffer]) -> Result { + let records = Records::encode(|index: usize| { + init::Region::new(objects[index].va(), objects[index].size()).map_err(|_| EINVAL) + })?; + objects[storage::ROOT].write(0, &records.root)?; + objects[RUNTIME_POINTERS].write(0, &records.runtime)?; + objects[CONTROL_REGION].write(0, &records.control) + } + pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } + pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDevice)->Result { + let value=self.objects[GLOBALS].read_u32(crate::m3_globals_debug::OFFSET)?; + dev_info!(dev.as_ref(),"M3_GLOBALS_DEBUG off={} value={} request_level={}\n", + u8::from(crate::m3_params::globals_debug_off()),value,crate::m3_params::consistent_perf()); + Ok(()) + } + pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result { + let Some(value)=crate::m3_consistent_perf::request(chip, + crate::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; + if value==0 && self.request_value==0 {return Ok(());} + let now=kernel::time::Instant::::now(); + let elapsed=self.request_check_at.map(|last|(now-last).as_nanos()); + if !crate::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { + return Ok(()); + } + type P=crate::fw::initdata::raw::PowerCtlBlock; + let at=core::mem::offset_of!(P,consistent_perf_state); + if !self.request_checked { + let mut flag=[0]; + self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut flag)?; + if flag[0]!=0 || self.objects[GLOBALS_POWER].read_u32(at)?!=0 {return Err(EINVAL);} + self.request_checked=true; + } + let observed=self.objects[GLOBALS_POWER].read_u32(at)?; + self.request_observed=observed; + if observed!=value { + self.objects[GLOBALS_POWER].packed_request(at,value)?; + agx_memory::publish(); + self.request_writes=self.request_writes.saturating_add(1); + } + self.request_value=value; + self.request_check_at=Some(now); + Ok(()) + } + /// Move the runtime cap by temperature (the thermal limit), then check the performance + /// state the firmware reports against the ceiling and the runtime cap. Called after boot and + /// after every job, by the job thread. + pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDevice, device: &crate::m3_device::Device, what: &str) -> Result { + let now = kernel::time::Instant::::now(); + if let Some(cap) = self.thermal.update(dev.as_ref(), now) { + crate::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; + } + let allowed = self.thermal.allowed(now); + let (over, peak) = self.pstates.check(dev.as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; + self.thermal.note_check(dev.as_ref(), now, over, peak)?; + self.thermal.busy(now); + Ok(()) + } + pub(crate) fn stats_region(&self) -> Result { + let owner = &self.objects[HARDWARE_DATA]; + init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL) + } + + pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDevice, + device: &crate::m3_device::Device, + sequence: u32) -> Result { + let cap = self.thermal.allowed(kernel::time::Instant::::now()); + self.pstates.log_active(dev.as_ref(), &mut self.objects[HARDWARE_DATA], + device.pstate_register()?, cap, sequence)?; + type P=crate::fw::initdata::raw::PowerCtlBlock; + type G=crate::fw::initdata::raw::GlobalsG15V14_8_3; + let request=self.objects[GLOBALS_POWER].read_u32(core::mem::offset_of!(P,consistent_perf_state))?; + let cap_word=self.objects[GLOBALS].read_u32(core::mem::offset_of!(G,perf_state_cap))?; + let power_if=core::mem::offset_of!(G,power_if_target); + let target1=self.objects[GLOBALS].read_u32(power_if+4)?; + let target2=self.objects[GLOBALS].read_u32(power_if+8)?; + let mut override_active=[0]; + self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut override_active)?; + dev_info!(dev.as_ref(),"M3_PERF_REQUEST sample={} level={} request={} cap_word={} target1={} target2={} writes={} observed={} override_active={}\n", + sequence,crate::m3_params::consistent_perf(),request,cap_word,target1,target2, + self.request_writes,self.request_observed,override_active[0]); + Ok(()) + } + + pub(crate) fn render_pb(&mut self) -> Result { + self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3_render::PB_PAGES).map_err(|_|EINVAL)?) + } + pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result { + if kind>2 {return Err(EINVAL);} + let state=kind*2;let ring=state+1; + let w=self.objects[state].read_u32(0x20)?; + if w>=256 || self.objects[state].read_u32(0)?!=w { return Err(EBUSY); } + let mut message=[0u8;0x18]; + message[8..16].copy_from_slice(&queue.to_le_bytes()); + message[16..20].copy_from_slice(&(kind as u32).to_le_bytes()); + message[20..22].copy_from_slice(&head.to_le_bytes()); + message[22]=event;message[23]=u8::from(new); // New queue; caller selected the event slot. + self.objects[ring].write(w as usize*0x18,&message)?; + agx_memory::publish(); + self.objects[state].u32(0x20,(w+1)%256)?; + agx_memory::publish(); + Ok(()) + } + /// Whether the firmware has consumed every message on pipe `kind` (0 TA, 1 3D, 2 compute). + pub(crate) fn pipe_free(&mut self, kind:usize) -> Result { + if kind>2 {return Err(EINVAL);} + Ok(self.objects[kind*2].read_u32(0)? == self.objects[kind*2].read_u32(0x20)?) + } + pub(crate) fn pipes_idle(&mut self) -> Result { + for state in [0,2,4] { + if self.objects[state].read_u32(0)? != self.objects[state].read_u32(0x20)? { + return Ok(false); + } + } + Ok(true) + } + pub(crate) fn ready(&mut self) -> Result { + // NewRegionB+4 is an input initialized to 1, not an acknowledgement. + // These are firmware-written transitions present in compute-15's + // pre-submit snapshot; both start at zero in the exported schema. + Ok(self.objects[RUNTIME_POINTERS].read_u32(0x314)? == 0xabcdabcd + && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1) + } + pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDevice) -> Result { + let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?; + let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?; + let a=self.objects[RUNTIME_FLAGS].read_u32(4)?; + let b=self.objects[RUNTIME_FLAGS].read_u32(0x14)?; + let c=self.objects[CONTROL_REGION].read_u32(0x45c4)?; + dev_info!(dev.as_ref(),"M3 firmware readiness: magic={:#x} runtime={} newA={}/{} newB={}\n",magic,state,a,b,c); + Ok(()) + } + /// The firmware's halt counter and its halted and resume flags. + pub(crate) fn halt_state(&mut self) -> Result<(u64,u32,u32)> { + let control=&mut self.objects[CONTROL_REGION]; + Ok((control.read_u64(0x4580)?,control.read_u32(0x4590)?,control.read_u32(0x45a0)?)) + } + /// Let a halted firmware continue: clear halted, then set resume. These + /// words keep the halt/resume layout of earlier Apple GPU firmware. + pub(crate) fn resume_halted(&mut self) -> Result { + let control=&mut self.objects[CONTROL_REGION]; + control.u32(0x4590,0)?; + agx_memory::publish(); + control.u32(0x45a0,1)?; + agx_memory::publish(); + Ok(()) + } + /// Read-only snapshot of the firmware's recovery words in the control + /// region (InitData+0xb0): the halt counter and the halted/resume flags + /// at +0x4580/+0x4590/+0x45a0, the firmware's own recovery information + /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0. + /// Nothing is acknowledged, cleared or resumed here. + pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDevice, events: u64) -> Result { + let control=&mut self.objects[CONTROL_REGION]; + let halt_count=control.read_u64(0x4580)?; + let halted=control.read_u32(0x4590)?; + let resume=control.read_u32(0x45a0)?; + let info=[control.read_u32(0x40a0)?,control.read_u32(0x40a4)?]; + let mut packet=[0u8;0x14]; + control.read(0x44a0,&mut packet)?; + dev_info!(dev.as_ref(),"M3 firmware recovery state: halt_count={} halted={} resume={} info={:#x}/{:#x} packet={:02x?} completed_events={} event_messages={}\n", + halt_count,halted,resume,info[0],info[1],packet,self.completed_events,events); + Ok(()) + } + /// Read-only: log the event ring entries from the host's read index to the firmware's write + /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the + /// completion count is not changed. + pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDevice) -> Result { + const SIZE: usize = 0x38; + const COUNT: usize = 256; + let r = self.objects[storage::EVENT*2].read_u32(0)? as usize; + let w = self.objects[storage::EVENT*2].read_u32(0x20)? as usize; + dev_info!(dev.as_ref(), "M3 event ring: read={} write={}\n", r, w); + if r >= COUNT || w >= COUNT { return Err(EIO); } + let mut index = r; + for _ in 0..16 { + if index == w { break; } + let mut raw = [0u8; SIZE]; + self.objects[storage::EVENT*2+1].read(index * SIZE, &mut raw)?; + dev_info!(dev.as_ref(), "M3 event ring entry {} kind={} bytes={:02x?}\n", + index, u32::from_le_bytes([raw[0], raw[1], raw[2], raw[3]]), raw); + index = (index + 1) % COUNT; + } + Ok(()) + } + pub(crate) fn control(&mut self, opcode: u32) -> Result { + if !matches!(opcode, 0x13 | 9) { return Err(EINVAL); } + let w = self.objects[storage::DEVICE_CONTROL*2].read_u32(0x20)?; + let r = self.objects[storage::DEVICE_CONTROL*2].read_u32(0)?; + if w >= 256 || r != w { return Err(EBUSY); } + let mut bytes = [0; 0x38]; + bytes[..4].copy_from_slice(&opcode.to_le_bytes()); + self.objects[storage::DEVICE_CONTROL*2+1].write(w as usize * 0x38, &bytes)?; + agx_memory::publish(); + let next = (w + 1) % 256; + self.objects[storage::DEVICE_CONTROL*2].u32(0x20, next)?; + agx_memory::publish(); + Ok(next) + } + pub(crate) fn control_done(&mut self, next: u32) -> Result { + Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next) + } + pub(crate) fn drain(&mut self, dev: &driver::AsahiDevice) -> Result { + // Event, trace, stats and six independent firmware log rings. + for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1), + (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1), + (storage::STATS*2,storage::STATS*2+1,0x40,256,1), + (storage::FIRMWARE_LOG*2,storage::FIRMWARE_LOG*2+1,init::fwlog::ENTRY_SIZE,init::fwlog::SLOTS,init::fwlog::CHANNELS)] { + for channel in 0..channels { + let base = channel * 0x30; + let mut r = self.objects[state].read_u32(base)? as usize; + let w = self.objects[state].read_u32(base+0x20)? as usize; + if r >= count || w >= count { return Err(EIO); } + let previous_read = r; + while r != w { + if (state==storage::TRACE*2 && self.completed_events<4) || state==storage::FIRMWARE_LOG*2 { + let mut raw=[0u8;0xd8]; + self.objects[ring].read((channel*count+r)*size,&mut raw[..size])?; + dev_info!(dev.as_ref(),"M3 fw-channel {} {:02x?}\n",state,&raw[..size]); + if state == storage::FIRMWARE_LOG*2 && self.objects[ring].read_u32((channel*count+r)*size)? == 2 { + let index = self.objects[ring].read_u64((channel*count+r)*size+8)?; + let offset = init::fwlog::payload_offset(channel,index).map_err(|_| EIO)?; + let mut payload = [0u8;init::fwlog::PAYLOAD_SIZE]; + self.objects[FWLOG_PAYLOAD].read(offset,&mut payload)?; + dev_info!(dev.as_ref(),"M3 fwlog payload {} {:02x?}\n",channel,payload); + } + } + if state == storage::EVENT*2 { + let kind = self.objects[ring].read_u32(r*size)?; + if kind == 1 { + let flags=self.objects[ring].read_u64(r*size+4)?; + self.completed_events+=u64::from((flags&7).count_ones()); + } + if kind == 0 || kind == 4 { + let mut raw=[0u8;0x38];self.objects[ring].read(r*size,&mut raw)?; + dev_err!(dev.as_ref(), "M3 firmware error event {} bytes={:02x?}\n", kind,raw); + return Err(EIO); + } + } + r = (r+1) % count; + } + // Publish actual consumption only, as RxChannel::get does. + // Empty polls need no write to firmware's shared state line. + if r != previous_read { self.objects[state].u32(base, r as u32)?; } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs new file mode 100644 index 00000000000000..1c2f0b1e8866da --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Owned M3 register apertures. The platform +//! power domain owns TVM/PMGR and stays on across probe/remove. Firmware and +//! queues must be stopped before this object is released. + +use kernel::{ + bindings, c_str, + device::Core, + devres::Devres, + error::to_result, + io::{ + mem::IoMem, + Io, + }, + platform, + prelude::*, + sync::aref::ARef, + time::{delay::fsleep, Delta, Instant, Monotonic}, +}; + +use crate::m3_firmware::Firmware; +use crate::m3_soc::Soc; + +const ASC_CPU_CONTROL: usize = 0x44; +const ASC_CPU_RUN: u32 = 1 << 4; + +pub(crate) struct Device { + dev: ARef, + asc: Pin>>>, + sgx: Pin>>, + firmware: Firmware, + core_mask: u64, + soc: &'static Soc, + power_vote: bool, +} + +impl Device { + /// `experiment` is the T8122 start experiment's values, on an armed T8122 only: it decides + /// the SGX setup write, which the T8122 table lacks. `t6031` is the same for an armed T6031. + pub(crate) fn new(pdev: &platform::Device, firmware: Firmware, soc: &'static Soc, + experiment: Option<&crate::t8122_start::Experiment>, + t6031: Option<&crate::t6031_start::Experiment>) -> Result { + // Map ASC before taking a vote. The larger SGX aperture contains + // this control window and the separate mailbox provider, so it is + // mapped without a conflicting claim over those child resources. + // Only the 16 KiB control window is used. Some device trees describe + // the ASC block as a larger range that also covers the mailbox, which + // the mailbox driver has already claimed, so do not claim the range. + let asc = KBox::pin_init( + pdev.io_request_by_name(c_str!("asc")) + .ok_or(EINVAL)? + .iomap_sized::<0x4000>(), + GFP_KERNEL, + )?; + dev_info!(pdev.as_ref(), "M3: ASC aperture mapped\n"); + let sgx = KBox::pin_init( + pdev.io_request_by_name(c_str!("sgx")) + .ok_or(EINVAL)? + .iomap(), + GFP_KERNEL, + )?; + dev_info!(pdev.as_ref(), "M3: SGX aperture mapped\n"); + let pmp = crate::m3_board::has_pmp_link(pdev); + let t8122 = crate::t8122_start::is_t8122(soc); + let t6031_soc = crate::t6031_start::is_t6031(soc); + if pmp && !t8122 && !t6031_soc { + dev_err!(pdev.as_ref(), "M3: optional apple,pmp GPU link is unsupported\n"); + return Err(ENOTSUPP); + } + if t8122 && !pmp { + dev_err!(pdev.as_ref(), "M3 G15G: current14 GPU requires its PMP supplier link\n"); + return Err(ENODEV); + } + if pmp { + // SAFETY: the consumer is live; the managed link pins its declared PMP supplier. + to_result(unsafe { bindings::apple_pmp_link_device(pdev.as_ref().as_raw()) })?; + } else { + dev_info!(pdev.as_ref(), "M3: no apple,pmp link; GPU power is left to its power domain\n"); + } + if asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + dev_err!( + pdev.as_ref(), + "M3 {}: ASC already running; refusing to take ownership\n", + soc.gpu_name + ); + return Err(EBUSY); + } + let mut device = Self { + dev: pdev.into(), + asc, + sgx, + firmware, + core_mask: 0, + soc, + power_vote: false, + }; + + if pmp { + // SAFETY: the linked current14 T8122 PMP owns AGX logical device 5. The bridge + // waits for firmware readiness and the power-command acknowledgement. + // An unacknowledged request may still have enabled AGX. Keep ownership so a + // failed acquisition releases it only after the ASC-stopped check in Drop. + device.power_vote = true; + to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) })?; + if t6031_soc { + dev_info!(pdev.as_ref(), "M3 G15C: PMP GPU device 5 power acknowledged\n"); + } else { + dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); + } + } + + let registers = device.sgx.access(pdev.as_ref())?; + let version = registers.try_read32(0xd04000)?; + let counts = registers.try_read32(0xd04010)?; + // The core-enable words: the second only on a SoC with more than 32 core slots. + let mut core_mask = u64::from(registers.try_read32(0xe01500)?); + if crate::m3_board::core_slots(soc) > 32 { + core_mask |= u64::from(registers.try_read32(0xe01504)?) << 32; + } + dev_info!(pdev.as_ref(), "M3 {}: power acknowledged, version={:#010x} counts={:#010x} core-mask={:#x}, firmware={}\n", + soc.gpu_name, version, counts, core_mask, device.firmware.version()); + // These words identify the qualified configuration of this SoC. The + // fused core mask varies with SKU; absent cores must never be enabled. + if version & soc.id.version_mask != soc.id.version + || counts & soc.id.counts_mask != soc.id.counts + || !crate::m3_board::core_mask_valid(soc, core_mask) + { + return Err(ENODEV); + } + let setup = match experiment { + Some(e) => { + match e.sgx_setup() { + Some((offset, value)) => dev_warn!(pdev.as_ref(), + "M3 G15G start: SGX setup write: SGX+{:#x} = {:#x} (asahi.t8122_sgx_setup=t6030)\n", offset, value), + None => dev_info!(pdev.as_ref(), + "M3 G15G start: no SGX setup write (asahi.t8122_sgx_setup=none)\n"), + } + e.sgx_setup() + } + None => match t6031 { + Some(e) => { + match e.sgx_setup() { + Some((offset, value)) => dev_warn!( + pdev.as_ref(), + "M3 G15C start: SGX setup write: SGX+{:#x} = {:#x}\n", + offset, + value + ), + None => dev_info!(pdev.as_ref(), "M3 G15C start: no SGX setup write\n"), + } + e.sgx_setup() + } + None => Some(soc.sgx_setup.ok_or(ENODEV)?), + }, + }; + if let Some((setup_offset, setup_value)) = setup { + registers.try_write32(setup_value, setup_offset)?; + } + device.core_mask = core_mask; + Ok(device) + } + + pub(crate) fn core_mask(&self) -> u64 { self.core_mask } + + /// The SoC table this device was admitted with. + pub(crate) fn soc(&self) -> &'static Soc { self.soc } + + pub(crate) fn check_idle(&self)->Result { + self.check_idle_parts(true,true) + } + /// [`Self::check_idle`] with its engine-busy (`busy`) and fault-bank (`faults`) register + /// accesses selectable (`asahi.m3_retire_mmio`); with neither it touches no GPU register. + pub(crate) fn check_idle_parts(&self,busy:bool,faults:bool)->Result { + if !busy && !faults {return Ok(());} + let sgx=self.sgx.try_access().ok_or(ENODEV)?; + if busy && (sgx.try_read64(0xc020)? | sgx.try_read64(0xc120)?) & 1 != 0 { + return Err(EBUSY); + } + if !faults {return Ok(());} + let selector=sgx.try_read64(0xd800)?; + let mut fault=0; + // One fault bank (selected through 0xd800) per GPU cluster: banks 0 and 1 on the + // two-cluster T6030, bank 0 alone on the single-cluster T8122. Never select a bank past + // the SoC's clusters. + for bank in 0..u64::from(self.soc.clusters) { + sgx.try_write64(bank,0xd800)?;fault|=sgx.try_read64(0xd8c0)?; + } + sgx.try_write64(selector,0xd800)?; + if fault!=0 {return Err(EIO);} Ok(()) + } + + pub(crate) fn check_drm(&self, pdev: &platform::Device) -> Result { + use kernel::dma::{Device as _, DmaMask}; + // SAFETY: This is the admitted M3 GPU's 42-bit physical DMA capability. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + crate::mmu::check_handoff_guard()?; + dev_info!(pdev.as_ref(), "M3 {}: GPU handoff lock checked.\n", self.soc.gpu_name); + for _ in 0..2 { + let drm: ARef = kernel::drm::Device::new( + pdev.as_ref(), crate::driver::AsahiData::new(pdev, None, true))?; + if drm.gpu().is_ok() { return Err(EIO); } + let mut object = crate::gem::new_kernel_object(&drm, 0x4000)?; + object.vmap()?.memset(0); + drop(object); + drop(drm); + } + dev_info!(pdev.as_ref(), "M3 {}: DRM data initialized; backend absent, device unregistered\n", self.soc.gpu_name); + Ok(()) + } + + /// The raw performance-state register (SGX+0xe01000). + pub(crate) fn pstate_register(&self) -> Result { + self.sgx.try_access().ok_or(ENODEV)?.try_read32(0xe01000) + } + + /// Read-only snapshot. Do not select or acknowledge fault banks here; + /// the firmware owns those registers until recovery is ported. + pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let pstate = sgx.try_read32(0xe01000)? & 0xf; + if pstate == 0 { + dev_info!(self.dev.as_ref(), "M3 {}: GPU is powered down; skipping engine/MMU register snapshot\n", self.soc.gpu_name); + return Ok(()); + } + dev_info!(self.dev.as_ref(), "M3 {}: reading engine/MMU snapshot at pstate={}\n", self.soc.gpu_name, pstate); + let selector=sgx.try_read64(0xd800)?; + for bank in [0,1,0x100,0x40000,0x40001,0x80000,0xc0000] { + sgx.try_write64(bank,0xd800)?; + let address=sgx.try_read64(0xd8c8)?; + let fault=sgx.try_read64(0xd8c0)?; + dev_info!(self.dev.as_ref(),"M3 bank {} fault={:#x} address_word={:#x} address={:#x}\n",bank,fault,address,address<<6); + if fault&1!=0 { + dev_info!(self.dev.as_ref(),"M3 fault mapping bank={} address={:#x} translation={:?}\n",bank,address<<6,vm.translate_iova(address<<6)); + } + } + sgx.try_write64(selector,0xd800)?; + // Read-only per-core service/debug snapshot, using the same + // core-selector protocol as the idle check. + let service=sgx.try_read32(0xa010)?; + let debug=sgx.try_read32(0xa000)?; + let mut cores=u64::from(sgx.try_read32(0xe01500)?); + if crate::m3_board::core_slots(self.soc) > 32 { + cores |= u64::from(sgx.try_read32(0xe01504)?) << 32; + } + for core in 0..crate::m3_board::core_slots(self.soc).min(64) { + if cores & (1u64< &Firmware { &self.firmware } + + pub(crate) fn require_stopped(&self, pdev: &platform::Device) -> Result { + if self.asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + return Err(EBUSY); + } + Ok(()) + } + + pub(crate) fn start_asc(&self, pdev: &platform::Device) -> Result { + self.require_stopped(pdev)?; + let asc = self.asc.access(pdev.as_ref())?; + asc.write32(asc.read32(ASC_CPU_CONTROL) | ASC_CPU_RUN, ASC_CPU_CONTROL); + Ok(()) + } + + pub(crate) fn stop_asc(&self) -> Result { + let asc = self.asc.try_access().ok_or(ENODEV)?; + asc.write32(asc.read32(ASC_CPU_CONTROL) & !ASC_CPU_RUN, ASC_CPU_CONTROL); + let start = Instant::::now(); + while asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + if start.elapsed() >= Delta::from_millis(100) { return Err(ETIMEDOUT); } + fsleep(Delta::from_micros(10)); + } + Ok(()) + } + +} + +impl Drop for Device { + fn drop(&mut self) { + if !self.power_vote { + return; + } + // ASC and firmware queues must be stopped before the inner GPU vote is relinquished. + match self.asc.try_access() { + Some(asc) if asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} + _ => { + dev_err!(self.dev.as_ref(), "M3 G15G: cannot prove ASC stopped; retaining PMP vote\n"); + return; + } + } + // SAFETY: the managed consumer link keeps the PMP bound through device removal. + if let Err(error) = to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 0) }) { + dev_err!(self.dev.as_ref(), "M3 G15G: PMP GPU power release failed: {:?}\n", error); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs new file mode 100644 index 00000000000000..25e65c6599084b --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M3 through the common Asahi GEM, GPUVM and render-node frontend. +use core::ops::Range; +use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}}; +use crate::{alloc, driver, drm_gpu::{DrmGpu, DrmGpuParams}, gem, gpu, hw, mmu, queue}; + +pub(crate) type Shared = Arc>>; +struct ProgressView(Arc); +impl kernel::debugfs::Writer for ProgressView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, completed, last_ns, healthy) = self.0.progress_snapshot(); + writeln!(f, "version=1 generation_ns={} completed={} last_completion_ns={} healthy={}", + generation, completed, last_ns, u8::from(healthy)) + } +} +// Separate endpoint preserves the strict completion-v1 ABI for old guards. +struct ActivityView(Arc); +impl kernel::debugfs::Writer for ActivityView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, epoch, healthy) = self.0.activity_snapshot(); + writeln!(f, "version=1 generation_ns={} epoch={} pending={} healthy={}", + generation, epoch, epoch & 1, u8::from(healthy)) + } +} + +struct TimingView(Arc); +impl kernel::debugfs::Writer for TimingView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (generation, _, _, healthy) = self.0.progress_snapshot(); + self.0.timing.write(f, generation, healthy) + } +} + +// Root-only writable debugfs policy; no mutable module-parameter references. +// Keep ownership independent of the runtime lock and remove the file on stop. +struct CompletionView(Arc); +impl kernel::debugfs::Writer for CompletionView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); + writeln!(f, "version=1 wait_us={} notifications={} sleeps={} timer_wakes={} cpu_preparations={}", + us, irqs, sleeps, timers, preparations) + } +} +impl kernel::debugfs::Reader for CompletionView { + fn read_from_slice(&self, reader: &mut kernel::uaccess::UserSliceReader) -> Result { + let len = reader.len(); + if len == 0 || len > 8 { return Err(EINVAL); } + let mut bytes = [0u8; 8]; + reader.read_slice(&mut bytes[..len])?; + let text = core::str::from_utf8(&bytes[..len]).map_err(|_| EINVAL)?; + let us = text.trim().parse::().map_err(|_| EINVAL)?; + self.0.set_wait_us(us) + } +} + +struct SubmitOverlapView(Arc); +impl kernel::debugfs::Writer for SubmitOverlapView { + fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + writeln!(f, "{}", u8::from(self.0.cpu_overlap())) + } +} +impl kernel::debugfs::Reader for SubmitOverlapView { + fn read_from_slice(&self, reader: &mut kernel::uaccess::UserSliceReader) -> Result { + let len=reader.len(); + if len==0 || len>2 {return Err(EINVAL);} + let mut bytes=[0u8;2];reader.read_slice(&mut bytes[..len])?; + let text=core::str::from_utf8(&bytes[..len]).map_err(|_|EINVAL)?.trim(); + match text { + "0" => self.0.set_cpu_overlap(false), + "1" => self.0.set_cpu_overlap(true), + _ => return Err(EINVAL), + } + Ok(()) + } +} + +pub(crate) struct Registered { + registration: Pin>>>>, + shared: Shared, + health: Arc, + _progress: Pin>>, + _activity: Pin>>, + _memory: Pin>>, + _timing: Pin>>, + _completion: Pin>>, + _overlap: Pin>>, +} +impl Registered { + pub(crate) fn start(pdev: &platform::Device, soc: &'static crate::m3_soc::Soc) -> Result { + let resources = crate::m3_resources::from_device(pdev, soc).inspect_err(|_| { + crate::t8122_start::not_admitted(pdev.as_ref(), soc); + crate::t6031_start::not_admitted(pdev.as_ref(), soc); + })?; + dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); + // An armed T8122 or T6031 start supplies the values its table leaves out. Any other SoC, + // or an unarmed T8122 or T6031, needs a complete table. + let experiment = crate::t8122_start::arm(pdev.as_ref(), soc)?; + let t6031 = crate::t6031_start::arm(pdev.as_ref(), soc)?; + if experiment.is_none() && t6031.is_none() { + soc.require_complete(pdev.as_ref())?; + } + let experiment = experiment.as_ref(); + let t6031 = t6031.as_ref(); + let refused = |stage: &'static str| { + move |e: &Error| { + crate::t8122_start::refused(pdev.as_ref(), experiment, stage, *e); + crate::t6031_start::refused(pdev.as_ref(), t6031, stage, *e); + } + }; + let firmware = crate::m3_firmware::identify_loaded(pdev, soc, resources, experiment, t6031) + .inspect_err(refused("the firmware identity"))?; + dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n"); + // InitData source and contents, before any GPU register access. + let contents = crate::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment, t6031) + .inspect_err(refused("the InitData contents"))?; + let device = crate::m3_device::Device::new(pdev, firmware, soc, experiment, t6031) + .inspect_err(refused("the GPU registers and identity"))?; + device.check_drm(pdev).inspect_err(refused("the DRM setup"))?; + let core_mask = device.core_mask(); + let max_frequency_khz = 1000 * contents.pstates.reported_max_mhz(); + let mut runtime = crate::m3_runtime::Runtime::new(pdev, device, contents)?; + runtime.boot(pdev)?; + dev_info!(pdev.as_ref(), "M3: runtime switches: asahi.m3_unlocked_wait={} asahi.m3_timeout_nohang={} resume-after-fault={}\n", + u8::from(crate::m3_params::unlocked_wait()), u8::from(crate::m3_params::timeout_nohang()), + u8::from(crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault))); + let drm = runtime.drm(); + let health = runtime.health(); + // Health holds no device references. The read-only file owns an Arc + // and is removed with registration; teardown latches failure first. + let directory = kernel::debugfs::Dir::new(c"asahi-m3"); + let progress = KBox::pin_init(directory.read_only_file(c"progress", + ProgressView(health.clone())), GFP_KERNEL)?; + let activity = KBox::pin_init(directory.read_only_file(c"activity", + ActivityView(health.clone())), GFP_KERNEL)?; + let memory = KBox::pin_init(directory.read_only_file(c"memory", + crate::agx_memory_stats::View), GFP_KERNEL)?; + let timing = KBox::pin_init(directory.read_only_file(c"timing", + TimingView(health.clone())), GFP_KERNEL)?; + let completion = KBox::pin_init(directory.read_write_file(c"completion_wait", + CompletionView(runtime.completion_wait())), GFP_KERNEL)?; + let overlap = KBox::pin_init(directory.read_write_file(c"submit_overlap", + SubmitOverlapView(runtime.completion_wait())), GFP_KERNEL)?; + let shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; + if !drm.completion.populate(crate::m3_completion::Completion::new(shared.clone())?) { + return Err(EBUSY); + } + let owner = Self { registration: KBox::pin_init(new_mutex!(None), GFP_KERNEL)?, shared: shared.clone(), health: health.clone(), _progress: progress, _activity: activity, _memory: memory, _timing: timing, _completion: completion, _overlap: overlap }; + let scheduler=Arc::new(drm::sched::Scheduler::new(drm.as_ref(),4,8,0,3000,kernel::c_str!("asahi_m3_sched"))?,GFP_KERNEL)?; + let backend: Arc = Arc::new(Backend { shared, health, scheduler, ids: gpu::SequenceIDs::default(), + core_mask, max_frequency_khz, soc }, GFP_KERNEL)?; + if !drm.gpu.populate(crate::drm_gpu::Backend::M3(backend)) { return Err(EBUSY); } + if crate::m3_board::expose_render_node(soc) { + *owner.registration.lock() = Some(drm::driver::Registration::new(&drm, 0)?); + dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); + } else { + dev_info!(pdev.as_ref(), "M3: firmware ready; render node not registered on this board (asahi.m3_expose=1 registers it)\n"); + } + Ok(owner) + } + pub(crate) fn stop(&self) { + self.health.mark_failed(); + drop(self.registration.lock().take()); + drop(self.shared.lock().take()); + } +} +impl Drop for Registered { fn drop(&mut self) { self.stop(); } } +struct Backend { shared: Shared, health: Arc, ids: gpu::SequenceIDs, + scheduler:Arc>, core_mask: u64, max_frequency_khz: u32, + soc: &'static crate::m3_soc::Soc } +impl DrmGpu for Backend { + fn init(&self) -> Result { if self.is_crashed() { Err(ENODEV) } else { Ok(()) } } + fn ids(&self) -> &gpu::SequenceIDs { &self.ids } + fn is_crashed(&self) -> bool { !self.health.healthy() } + fn update_globals(&self) {} + + fn supports_vm_status(&self) -> bool { true } + fn supports_scheduled_queues(&self)->bool {true} + fn submission_error(&self) -> i32 { if self.is_crashed() { EIO.to_errno() } else { 0 } } + fn params(&self) -> Result { + let soc = self.soc; + let masks = crate::m3_board::core_masks(soc, self.core_mask); + Ok(DrmGpuParams { gpu_generation: hw::GpuGen::G15 as u32, + gpu_variant: soc.gpu_variant as u32, gpu_revision: soc.gpu_revision as u32, + chip_id: soc.chip_id, num_dies: 1, num_clusters_total: soc.clusters, + num_cores_per_cluster: soc.cores_per_cluster, + core_masks: masks, max_frequency_khz: self.max_frequency_khz, usc_generation: 3, + gpu_hal_generation: hw::GpuHalGeneration::Legacy as u32, + max_commands_per_submission: crate::file::MAX_COMMANDS_PER_SUBMISSION }) + } + fn user_range(&self) -> Result> { Ok(mmu::UAT_PGSZ as u64..0x2ff_ffff_8000) } + fn unknown_page(&self) -> Result { Ok(0x2ff_ffff_8000) } + fn base_clock_hz(&self) -> u32 { + let value: u64; + // SAFETY: read-only architectural counter frequency. + unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x=out(reg) value) }; + value as u32 + } + fn new_vm(&self, range: Range) -> Result { + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range) + } + fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, + _ualloc_priv: Arc>, priority: u32, + usc_exec_base: u64) -> Result> { + Ok(KBox::new(crate::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, + self.soc.features.fragment_dependency)?,GFP_KERNEL)?) + } + fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { + self.shared.lock().as_ref().ok_or(ENODEV)?.map_timestamp(bo,range) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs new file mode 100644 index 00000000000000..9ae509ad959c46 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + + +use crate::m3_resources as agx_resources; + +/// A GPU firmware image the runtime can identify. +#[derive(Debug)] +pub(crate) struct KnownImage { + /// Human-readable identity, for logs. + pub(crate) name: &'static str, + uuid: Option<[u8; 16]>, + stkg_sha256: Option<[u8; 32]>, + /// The validated InitData magic (InitData+0), or `None` until runtime admission is supported. + pub(crate) initdata_magic: Option, +} + +impl KnownImage { + /// Runtime admission requires a validated InitData version and all recorded identities. + pub(crate) fn accepts_runtime(&self, uuid: Option<[u8; 16]>, digest: &[u8; 32]) -> bool { + self.initdata_magic.is_some() + && (self.uuid.is_some() || self.stkg_sha256.is_some()) + && self.uuid.map_or(true, |known| uuid == Some(known)) + && self.stkg_sha256.map_or(true, |known| known == *digest) + } +} + +/// T6030 GPU firmware images known to the M3 runtime. An image is accepted when every recorded +/// identity (UUID, hash) matches. +pub(crate) static KNOWN_IMAGES: [KnownImage; 2] = [ + KnownImage { + name: "J514S RTKit-2419.140.12", + uuid: None, + stkg_sha256: Some(crate::m3_firmware::TEXT_SHA256), + initdata_magic: Some(G15_V14_8_3_INITDATA), + }, + KnownImage { + name: "g15s build b0 (firmware 14.8.3) RTKit-2419.140.12", + uuid: Some([ + 0xdb, 0xf3, 0x7c, 0x40, 0xea, 0xd5, 0x37, 0x60, 0x94, 0x41, 0x99, 0x50, + 0x18, 0x78, 0x29, 0x55, + ]), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), + }, +]; + +/// The G15 InitData version of firmware-compat 14.8.3: the version of both T6030 images and of +/// the T8122 image. +pub(crate) const G15_V14_8_3_INITDATA: u64 = 0x0c08_e21e_8380_0490; + +/// The image-info UUID of the T8122 C0 firmware-compat 14.8.3 image. +const T8122_C0_UUID: [u8; 16] = [ + 0xdf, 0x69, 0x7f, 0x05, 0xf6, 0xb5, 0x33, 0xef, 0xa1, 0x37, 0x61, 0xc4, 0xa7, 0x3d, 0x66, 0x6a, +]; + +/// T8122 C0, firmware-compat 14.8.3: the image the macOS 14.8.3 system firmware loads on the +/// M3 MacBook Airs. It takes the G15 14.8.3 InitData version. +pub(crate) static KNOWN_IMAGES_T8122: [KnownImage; 1] = [KnownImage { + name: "T8122 C0 firmware-compat 14.8.3", + uuid: Some(T8122_C0_UUID), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), +}]; + +/// The same T8122 image for the start experiment (`t8122_start`, only with +/// `asahi.t8122_start=1`), where `asahi.t8122_initdata_version` may replace the version. +pub(crate) static KNOWN_IMAGES_T8122_EXPERIMENT: [KnownImage; 1] = [KnownImage { + name: "T8122 C0 firmware-compat 14.8.3 (start experiment)", + uuid: Some(T8122_C0_UUID), + stkg_sha256: None, + initdata_magic: Some(G15_V14_8_3_INITDATA), +}]; + +const IMAGE_INFO_OFFSET: usize = 0x4200; +/// Size of the identifying part of the image-info header. +const IMAGE_INFO_SIZE: usize = 0x38; +/// First word of the header: a branch over it (`b +0x44`). +const IMAGE_INFO_BRANCH: u32 = 0x1400_0011; +/// Header magic, "uuid". +const IMAGE_INFO_MAGIC: u32 = 0x6469_7575; + +/// Parsed image-info header: UUID, patchbay offset and size, TEXT size. +pub(crate) struct ImageInfo { + pub(crate) uuid: [u8; 16], + pub(crate) patchbay: core::ops::Range, + pub(crate) text_size: usize, +} + +pub(crate) fn image_info(text: &[u8]) -> Option { + let hdr = text.get(IMAGE_INFO_OFFSET..IMAGE_INFO_OFFSET + IMAGE_INFO_SIZE)?; + let word = |i: usize| u32::from_le_bytes([hdr[i], hdr[i + 1], hdr[i + 2], hdr[i + 3]]); + if word(0) != IMAGE_INFO_BRANCH || word(4) != IMAGE_INFO_MAGIC || word(8) != 5 { + return None; + } + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&hdr[0x14..0x24]); + let start = word(0x2c) as usize; + let end = start.checked_add(word(0x30) as usize)?; + if word(0x34) as usize != text.len() { + return None; + } + text.get(start..end)?; + Some(ImageInfo { + uuid, + patchbay: start..end, + text_size: word(0x34) as usize, + }) +} + +/// The layout of a loaded GPU firmware the runtime accepts: segment sizes and VAs, and the +/// per-boot words of the text segment that identification ignores. +#[derive(Debug)] +pub(crate) struct Layout { + pub(crate) text_size: u64, + pub(crate) data_size: u64, + pub(crate) vas: [u64; 2], + /// Offset of the 8-byte tag that precedes the 8 per-boot bytes zeroed before hashing. + pub(crate) entropy_tag_offset: usize, + pub(crate) entropy_tag: [u8; 8], + /// The firmware version, for logs. + pub(crate) version: &'static str, +} + +pub(crate) static T6030_LAYOUT: Layout = Layout { + text_size: 0x5c000, + data_size: 0x114000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0005_c000], + entropy_tag_offset: 0x58e88, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "RTKit-2419.140.12.release", +}; + +/// T8122 firmware-compat 14.8.3 segment sizes, virtual addresses and entropy-tag bounds. +pub(crate) static T8122_LAYOUT: Layout = Layout { + text_size: 0x64000, + data_size: 0xb0000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], + entropy_tag_offset: 0x5d54a, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "RTKit-2419.140.12.release", +}; + +/// The T6031 firmware-compat 14.8.3 segment sizes and VAs, from the J516C ADT (iop-gfx-nub0 +/// segment-ranges: TEXT 0x64000 and DATA 0x140000 at the VAs both other layouts use). Neither +/// existing layout matches them. The boot-entropy tag offset is not known (0 stands for it); +/// an armed start searches the loaded text for the tag. The loaded image's version string is +/// not known ("unknown" stands for it). +pub(crate) const T6031_LAYOUT: Layout = Layout { + text_size: 0x64000, + data_size: 0x14_0000, + vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], + entropy_tag_offset: 0, + entropy_tag: *b"GKTS\x08\x00\x00\x00", + version: "unknown", +}; + +pub(crate) const TEXT_SHA256: [u8; 32] = [ + 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; + +fn normalize_boot_entropy(layout: &Layout, text: &mut [u8]) -> bool { + let tag = layout.entropy_tag_offset; + if text.len() != layout.text_size as usize { + return false; + } + let Some(end) = tag.checked_add(16) else { return false; }; + let Some(entropy) = text.get_mut(tag..end) else { return false; }; + if entropy[..8] != layout.entropy_tag { + return false; + } + entropy[8..].fill(0); + true +} + +#[derive(Debug)] +pub(crate) struct Firmware { + pub(crate) resources: agx_resources::Resources, + /// The InitData version validated for the identified image. + pub(crate) initdata_magic: u64, + /// The accepted layout. + pub(crate) layout: &'static Layout, +} + +impl Firmware { + pub(crate) const fn version(&self) -> &'static str { + self.layout.version + } + + fn matching_layout(layout: &Layout, resources: &agx_resources::Resources) -> bool { + resources.regions[4].size == layout.text_size + && resources.regions[5].size == layout.data_size + && resources.firmware_vas == layout.vas + } +} + +#[cfg(not(test))] +pub(crate) fn identify_loaded( + pdev: &kernel::platform::Device, + soc: &'static crate::m3_soc::Soc, + resources: agx_resources::Resources, + experiment: Option<&crate::t8122_start::Experiment>, + t6031: Option<&crate::t6031_start::Experiment>, +) -> kernel::error::Result { + use kernel::{ + bindings, c_str, + io::mem::{Mem, MemFlag}, + prelude::*, + }; + + let layout = soc.firmware.ok_or(ENODEV)?; + if !Firmware::matching_layout(layout, &resources) { + dev_err!(pdev.as_ref(), "M3 {}: unsupported firmware segment layout\n", soc.gpu_name); + return Err(ENODEV); + } + let node = pdev.as_ref().of_node().ok_or(ENODEV)?; + let text = agx_resources::reserved_resource(&node, c_str!("fw-text"))?; + if text.start() != resources.regions[4].base || text.size() != layout.text_size { + return Err(EINVAL); + } + let mapping = unsafe { Mem::try_new(text, MemFlag::WB.into()) }?; + let bytes = unsafe { core::slice::from_raw_parts(mapping.ptr(), mapping.size()) }; + let mut canonical = KVec::new(); + canonical.extend_from_slice(bytes, GFP_KERNEL)?; + // An armed T6031 searches the loaded text for its boot-entropy tag. The layout's offset is + // a placeholder, so the T8122 image table must not see this image. + if let Some(experiment) = t6031 { + experiment.normalize_text(&mut canonical)?; + let mut digest = [0u8; 32]; + // SAFETY: the initialized private copy and distinct digest live for the synchronous + // SHA-256 call. Normalization never writes the loaded firmware. + unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; + experiment.admit_image(pdev.as_ref(), bytes, &digest)?; + return Ok(Firmware { + resources, + initdata_magic: experiment.initdata_version(), + layout, + }); + } + if !normalize_boot_entropy(layout, &mut canonical) { + return Err(ENODEV); + } + let mut digest = [0u8; 32]; + // SAFETY: the initialized private copy and distinct digest live for the + // synchronous SHA-256 call. Normalization never writes loaded firmware. + unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; + // The T8122 start experiment identifies the same image, with its own InitData version. + let images = match experiment { + Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], + None => soc.images, + }; + let image = crate::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; + let initdata_magic = match experiment { + Some(experiment) => experiment.initdata_version(), + None => image.initdata_magic.ok_or(ENODEV)?, + }; + Ok(Firmware { resources, initdata_magic, layout }) +} + +#[cfg(test)] +mod tests { + use super::*; + use agx_resources::{Region, Resources}; + + fn image_header(uuid: [u8; 16]) -> Vec { + let mut text = vec![0; T6030_LAYOUT.text_size as usize]; + let size = text.len() as u32; + for (offset, value) in [ + (0, IMAGE_INFO_BRANCH), (4, IMAGE_INFO_MAGIC), (8, 5), + (0x2c, size - 32), (0x30, 32), (0x34, size), + ] { + header_word(&mut text, offset, value); + } + text[IMAGE_INFO_OFFSET + 0x14..IMAGE_INFO_OFFSET + 0x24].copy_from_slice(&uuid); + text + } + + fn header_word(text: &mut [u8], offset: usize, value: u32) { + let start = IMAGE_INFO_OFFSET + offset; + text[start..start + 4].copy_from_slice(&value.to_le_bytes()); + } + + fn admits_text(image: &KnownImage, text: &[u8], digest: &[u8; 32]) -> bool { + image.accepts_runtime(image_info(text).map(|info| info.uuid), digest) + } + + #[test] + fn uuid_admission_requires_header_bounds_and_exact_text_size() { + let image = &KNOWN_IMAGES[1]; + let text = image_header(image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + let size = text.len() as u32; + for (offset, value) in [ + (0x34, 0), (0x34, size - 1), (0x34, size + 1), + (0x2c, size), (0x2c, u32::MAX), (0x30, u32::MAX), + ] { + let mut bad = text.clone(); + header_word(&mut bad, offset, value); + assert!(!admits_text(image, &bad, &[0; 32]), "offset {offset:#x}, {value:#x}"); + } + let info = image_info(&text).unwrap(); + assert_eq!(info.patchbay, text.len() - 32..text.len()); + assert_eq!(info.text_size, text.len()); + } + + #[test] + fn uuid_admission_rejects_bad_header_words_and_truncation() { + let image = &KNOWN_IMAGES[1]; + let text = image_header(image.uuid.unwrap()); + for offset in [0, 4, 8] { + let mut bad = text.clone(); + header_word(&mut bad, offset, 0); + assert!(!admits_text(image, &bad, &[0; 32])); + } + for len in [0, IMAGE_INFO_OFFSET, IMAGE_INFO_OFFSET + IMAGE_INFO_SIZE - 1, text.len() - 1] { + assert!(!admits_text(image, &text[..len], &[0; 32])); + } + } + + #[test] + fn runtime_admission_requires_every_recorded_identity() { + let image = KnownImage { + name: "test", uuid: Some([1; 16]), stkg_sha256: Some([2; 32]), + initdata_magic: Some(3), + }; + let text = image_header([1; 16]); + assert!(admits_text(&image, &text, &[2; 32])); + assert!(!admits_text(&image, &text, &[0; 32])); + assert!(!admits_text(&image, &image_header([0; 16]), &[2; 32])); + assert!(!image.accepts_runtime(None, &[2; 32])); + let anonymous = KnownImage { uuid: None, stkg_sha256: None, ..image }; + assert!(!admits_text(&anonymous, &text, &[2; 32])); + } + + #[test] + fn digest_only_admission_preserves_exact_digest_requirement() { + let image = &KNOWN_IMAGES[0]; + assert!(admits_text(image, &[], &TEXT_SHA256)); + let mut wrong = TEXT_SHA256; + wrong[0] ^= 1; + assert!(!admits_text(image, &[], &wrong)); + } + + #[test] + fn t8122_admission_requires_qualified_initdata_and_matching_image_identity() { + let image = &KNOWN_IMAGES_T8122[0]; + assert_eq!(image.initdata_magic, Some(G15_V14_8_3_INITDATA)); + let text = image_header(image.uuid.unwrap()); + assert_eq!(image_info(&text).unwrap().uuid, image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + + // A qualified version does not admit an unknown or another chip's image. + for wrong_uuid in [[0; 16], KNOWN_IMAGES[1].uuid.unwrap()] { + assert!(!admits_text(image, &image_header(wrong_uuid), &[0; 32])); + } + assert!(!image.accepts_runtime(None, &[0; 32])); + + // Matching identity still cannot admit an unqualified InitData layout. + let unqualified = KnownImage { + name: image.name, uuid: image.uuid, stkg_sha256: image.stkg_sha256, + initdata_magic: None, + }; + assert!(!admits_text(&unqualified, &text, &[0; 32])); + let unidentified = KnownImage { + name: "unknown", uuid: None, stkg_sha256: None, + initdata_magic: image.initdata_magic, + }; + assert!(!admits_text(&unidentified, &text, &[0; 32])); + } + + #[test] + fn t8122_experiment_record_identifies_the_same_image_with_the_g15_version() { + let image = &KNOWN_IMAGES_T8122_EXPERIMENT[0]; + assert_eq!(image.uuid, KNOWN_IMAGES_T8122[0].uuid); + assert_eq!(image.stkg_sha256, None); + assert_eq!(image.initdata_magic, Some(G15_V14_8_3_INITDATA)); + assert!(KNOWN_IMAGES.iter().all(|i| i.initdata_magic == Some(G15_V14_8_3_INITDATA))); + let text = image_header(image.uuid.unwrap()); + assert!(admits_text(image, &text, &[0; 32])); + assert!(!admits_text(image, &image_header(KNOWN_IMAGES[1].uuid.unwrap()), &[0; 32])); + assert!(!admits_text(&KNOWN_IMAGES[1], &text, &[0; 32])); + } + + #[test] + fn each_layout_rejects_the_other_chips_segments() { + for (layout, other) in [(&T6030_LAYOUT, &T8122_LAYOUT), (&T8122_LAYOUT, &T6030_LAYOUT)] { + let mut regions = [Region { base: 0, size: 0 }; 6]; + regions[4].size = layout.text_size; + regions[5].size = layout.data_size; + let mut resources = Resources { regions, firmware_vas: layout.vas }; + assert!(Firmware::matching_layout(layout, &resources)); + assert!(!Firmware::matching_layout(other, &resources)); + resources.firmware_vas[1] += 0x4000; + assert!(!Firmware::matching_layout(layout, &resources)); + } + } + + #[test] + fn t6031_layout_is_the_adt_segments_and_matches_neither_other_layout() { + let layout = &T6031_LAYOUT; + assert_eq!((layout.text_size, layout.data_size), (0x64000, 0x140000)); + assert_eq!(layout.vas, [0xffff_fc00_0000_0000, 0xffff_fc00_0000_0000 + 0x64000]); + assert_eq!(layout.entropy_tag, T8122_LAYOUT.entropy_tag); + let mut regions = [Region { base: 0, size: 0 }; 6]; + regions[4].size = layout.text_size; + regions[5].size = layout.data_size; + let resources = Resources { regions, firmware_vas: layout.vas }; + assert!(Firmware::matching_layout(layout, &resources)); + assert!(!Firmware::matching_layout(&T6030_LAYOUT, &resources)); + assert!(!Firmware::matching_layout(&T8122_LAYOUT, &resources)); + } + + #[test] + fn entropy_normalization_changes_only_the_eight_boot_bytes() { + for layout in [&T6030_LAYOUT, &T8122_LAYOUT] { + let mut text = vec![0xa5; layout.text_size as usize]; + let tag = layout.entropy_tag_offset; + text[tag..tag + 8].copy_from_slice(&layout.entropy_tag); + let before = text.clone(); + assert!(normalize_boot_entropy(layout, &mut text)); + assert_eq!(&text[..tag + 8], &before[..tag + 8]); + assert_eq!(&text[tag + 8..tag + 16], &[0; 8]); + assert_eq!(&text[tag + 16..], &before[tag + 16..]); + } + } + + #[test] + fn bad_entropy_bounds_tags_and_lengths_do_not_mutate_text() { + for offset in [8, 16, usize::MAX] { + let layout = Layout { text_size: 16, entropy_tag_offset: offset, ..T8122_LAYOUT }; + let mut text = vec![0xa5; 16]; + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 16]); + } + let layout = Layout { text_size: 16, entropy_tag_offset: 0, ..T8122_LAYOUT }; + let mut text = vec![0xa5; 16]; + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 16]); + text.pop(); + assert!(!normalize_boot_entropy(&layout, &mut text)); + assert_eq!(text, vec![0xa5; 15]); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs new file mode 100644 index 00000000000000..cbfb771fd08734 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Owned firmware/engine buffers with the qualified M3 page offsets. +use kernel::prelude::*; +use crate::{driver, gem, mmu, pgtable::prot}; + +enum Backing { + Coherent(kernel::dma::Coherent<[u8]>), + Paged(gem::ObjectRef), +} + +pub(crate) struct Buffer { + mapping: mmu::KernelMapping, + gpu_mapping: Option, + // Every client view must preserve the owner's cache and access policy. + // A VM switch changes the address space, not the backing's attributes. + gpu_protection: prot::Prot, + object: Backing, + offset: usize, + size: usize, + _coherent_allocation: Option, +} + +impl Buffer { + pub(crate) fn at(dev: &driver::AsahiDevice, fw: &mmu::Vm, + gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, + gpu_shared: bool) -> Result { + Self::at_prot(dev,fw,gpu,address,size, + if gpu_shared {prot::PROT_GPU_FW_SHARED_RW} else {prot::PROT_FW_SHARED_RW}, + prot::PROT_GPU_SHARED_RW) + } + pub(crate) fn at_prot(dev: &driver::AsahiDevice, fw: &mmu::Vm, + gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, + protection: prot::Prot, gpu_protection: prot::Prot) -> Result { + let offset=(address.or(gpu.map(|(_,a)|a)).unwrap_or(0)&0x3fff) as usize; + if size==0 || gpu.is_some_and(|(_,a)| a&0x3fff!=offset as u64) {return Err(EINVAL);} + let allocated=(size.checked_add(offset).and_then(|n|n.checked_add(0x3fff)).ok_or(EOVERFLOW)?)&!0x3fff; + // The DMA core tries the default CMA area first and falls back to the + // page allocator. The graphs outgrow that area, so a CMA miss is the + // normal case here and is not reported; only a failed fallback is. + // SAFETY: M3 runtime owns the bound platform device for the entire + // diagnostic; all mappings retire before normal DMA allocation Drop. + let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_bound()},allocated,GFP_KERNEL, + kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?; + let pa:usize=object.dma_handle().try_into()?; + let mapping=if let Some(a)=address {fw.map_io(a&!0x3fff,pa,allocated,protection)?} + else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?}; + let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; + Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Coherent(object),offset,size, + _coherent_allocation:Some(crate::agx_memory_stats::Allocation::coherent(allocated)?)}) + } + /// Allocate owned tiler storage in the driver-reserved GPU range. Keep the + /// old allocation mapped until the replacement has been fully allocated. + pub(crate) fn tiler(dev:&driver::AsahiDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { + if size==0 || size.checked_add(mmu::UAT_PGMSK).is_none() {return Err(EINVAL);} + // Large render targets can require 128 MiB of tiler storage. Engines + // and firmware address it through UAT, so physically contiguous DMA + // backing is unnecessary and can exceed the entire CMA reservation. + // GEM provides zeroed pages and the same WC CPU/device coherency used + // by M4. Keep both mappings ahead of their backing in drop order. + let mut object=gem::new_kernel_object_wc(dev,size)?; + let mapping=object.map_into_range(fw,0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000, + mmu::UAT_PGSZ as u64,prot::PROT_GPU_FW_SHARED_RW,true)?; + // Growth retains the cached GPU-only policy of the original TPC and + // tilemap allocations; the firmware alias remains shared/uncached. + let gpu_protection=prot::Prot::from_pte(crate::m3_pass_layout::Access::GpuCachedRw.pte()); + let gpu_mapping=Some(object.map_into_range(vm,0x74_0000_0000..0x78_0000_0000, + mmu::UAT_PGSZ as u64,gpu_protection,true)?); + Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Paged(object),offset:0,size,_coherent_allocation:None}) + } + pub(crate) fn va(&self)->u64 {self.mapping.iova()+self.offset as u64} + pub(crate) fn gpu_va(&self)->Result {Ok(self.gpu_mapping.as_ref().ok_or(EINVAL)?.iova()+self.offset as u64)} + pub(crate) fn size(&self)->usize {self.size} + pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result { + let address=self.gpu_va()?&!0x3fff; + match &mut self.object { + Backing::Coherent(object)=>vm.map_io(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), + Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true), + } + } + /// The backing retains its CPU mapping until Buffer is dropped. Callers + /// check the range and do not let this pointer escape the current access. + fn cpu_ptr(&mut self)->Result<*mut u8> { + match &mut self.object { + Backing::Coherent(object)=>Ok(object.as_mut_ptr().cast::()), + Backing::Paged(object)=>Ok(object.cached_vmap()?.as_mut_ptr()), + } + } + + pub(crate) fn swap_gpu_view(&mut self, mapping: mmu::KernelMapping)->Result { + self.gpu_mapping.replace(mapping).ok_or(EINVAL) + } + pub(crate) fn write(&mut self,offset:usize,data:&[u8])->Result { + if offset.checked_add(data.len()).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + unsafe {core::ptr::copy_nonoverlapping(data.as_ptr(),ptr.add(self.offset+offset),data.len())}; Ok(()) + } + /// Initialize the complete owned span through its retained CPU mapping. + /// The caller must own the storage or have retired all device consumers. + pub(crate) fn fill(&mut self, value: u8) -> Result { + self.fill_range(0,self.size,value) + } + /// Reset one unpublished slot without touching other queued device owners. + pub(crate) fn fill_range(&mut self,offset:usize,size:usize,value:u8)->Result { + if offset.checked_add(size).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr = self.cpu_ptr()?; + // SAFETY: allocation validates offset + size against its backing, and + // cpu_ptr retains that entire coherent/WC mapping for this access. + // Resolve the mapping once, as the G16 buffer clear path does, instead + // of repeating cached_vmap and memcpy for every 256-byte chunk. + unsafe { core::ptr::write_bytes(ptr.add(self.offset+offset), value, size) }; + Ok(()) + } + pub(crate) fn u32(&mut self,offset:usize,value:u32)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + if start&3!=0 {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: aligned, bounded WC GEM storage retained by this owner. + unsafe {ptr.add(start).cast::().write_volatile(value.to_le())};Ok(()) + } + pub(crate) fn u64(&mut self,offset:usize,value:u64)->Result {self.write(offset,&value.to_le_bytes())} + /// Publish the unaligned performance-request word with bounded byte stores. + pub(crate) fn packed_request(&mut self,offset:usize,value:u32)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: checked span in retained CPU storage; no aligned access is required. + unsafe {crate::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; + Ok(()) + } + pub(crate) fn read(&mut self,offset:usize,data:&mut[u8])->Result { + if offset.checked_add(data.len()).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let ptr=self.cpu_ptr()?; + // SAFETY: checked WC GEM span; firmware may change the shared bytes. + unsafe { + core::arch::asm!("dmb osh",options(nostack,preserves_flags)); + for (i,b) in data.iter_mut().enumerate() {*b=ptr.cast_const().add(self.offset+offset+i).read_volatile();} + } Ok(()) + } + pub(crate) fn read_u32(&mut self,offset:usize)->Result { + if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + if start&3!=0 {let mut b=[0;4];self.read(offset,&mut b)?;return Ok(u32::from_le_bytes(b));} + let ptr=self.cpu_ptr()?; + // SAFETY: aligned, checked atomic producer/consumer word in WC RAM. + let value=unsafe {ptr.cast_const().add(start).cast::().read_volatile()}; + unsafe {core::arch::asm!("dmb oshld",options(nostack,preserves_flags))}; + Ok(u32::from_le(value)) + } + pub(crate) fn read_u64(&mut self,offset:usize)->Result { + if offset.checked_add(8).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} + let start=self.offset.checked_add(offset).ok_or(EOVERFLOW)?; + // Packed firmware records also contain unaligned u64 fields. Retain + // their byte-wise access rather than issuing an unaligned word load. + if start&7!=0 {let mut b=[0;8];self.read(offset,&mut b)?;return Ok(u64::from_le_bytes(b));} + let ptr=self.cpu_ptr()?; + // SAFETY: retained page-aligned coherent/WC backing, checked aligned + // eight-byte span. One volatile word load avoids eight separate reads + // and byte-wise tearing of an aligned firmware timestamp/counter. + let value=unsafe {ptr.cast_const().add(start).cast::().read_volatile()}; + unsafe {core::arch::asm!("dmb oshld",options(nostack,preserves_flags))}; + Ok(u64::from_le(value)) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs new file mode 100644 index 00000000000000..d34e1f23840eb8 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -0,0 +1,900 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module parameters of the T6030 (M3) backends. +//! +//! The `module!` parameter support only parses plain integers. The parameters here are declared +//! with their own `kernel_param` entries (the same layout the `module!` macro emits), so they can +//! take symbolic values on the kernel command line, e.g. `asahi.m3_backend=manager`, and hex or +//! decimal masks. + +use core::ffi::{ + c_char, + c_int, +}; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use kernel::{ + device::Core, + platform, + prelude::*, // +}; + +/// Parse a decimal or `0x`-prefixed hexadecimal `u64`. +fn parse_u64(text: &str) -> Option { + let text = text.trim(); + match text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) { + Some(hex) => u64::from_str_radix(hex, 16).ok(), + None => text.parse::().ok(), + } +} + +/// Store the parsed value of a parameter string into the `AtomicU64` behind `kp`. +/// +/// # Safety +/// +/// `val` must be NULL or a NUL-terminated string, and `kp` must be a `kernel_param` declared by +/// [`m3_param!`], whose `arg` points at a static `AtomicU64`. +unsafe fn set_param( + val: *const c_char, + kp: *const kernel::bindings::kernel_param, + parse: fn(&str) -> Option, +) -> c_int { + if val.is_null() { + return EINVAL.to_errno(); + } + // SAFETY: `val` is a NUL-terminated string per the function contract. + let Ok(text) = unsafe { core::ffi::CStr::from_ptr(val) }.to_str() else { + return EINVAL.to_errno(); + }; + let Some(value) = parse(text) else { + return EINVAL.to_errno(); + }; + // SAFETY: `arg` points at the static `AtomicU64` this entry was declared with. + let slot = unsafe { &*(*kp).__bindgen_anon_1.arg.cast::() }; + slot.store(value, Ordering::Relaxed); + 0 +} + +/// Declare a module parameter `asahi.` stored in the static `AtomicU64` `$storage` and +/// parsed by `$parse: fn(&str) -> Option`. Not visible in sysfs. +macro_rules! m3_param { + ($name:literal, $storage:ident, $parse:path) => { + m3_param!($name, $storage, $parse, 0, None); + }; + ($name:literal, $storage:ident, $parse:path, $perm:expr, $get:expr) => { + const _: () = { + unsafe extern "C" fn set( + val: *const c_char, + kp: *const kernel::bindings::kernel_param, + ) -> c_int { + // SAFETY: Called by the parameter parser with a value string (or NULL) and the + // `kernel_param` entry declared below. + unsafe { set_param(val, kp, $parse) } + } + + static OPS: kernel::bindings::kernel_param_ops = kernel::bindings::kernel_param_ops { + flags: 0, + set: Some(set), + get: $get, + free: None, + }; + + #[link_section = "__param"] + #[used(compiler)] + static PARAM: kernel::module_param::KernelParam = + kernel::module_param::KernelParam::new(kernel::bindings::kernel_param { + name: kernel::str::as_char_ptr_in_const_context(if cfg!(MODULE) { + kernel::c_str!($name) + } else { + kernel::c_str!(concat!("asahi.", $name)) + }), + // `__this_module` is constructed by the kernel at load time and is not + // freed until the module is unloaded. Take its address without creating a + // reference to the mutable static. + #[cfg(MODULE)] + mod_: (&raw const kernel::bindings::__this_module).cast_mut(), + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: core::ptr::from_ref(&OPS), + perm: $perm, + level: -1, + flags: 0, + __bindgen_anon_1: kernel::bindings::kernel_param__bindgen_ty_1 { + arg: core::ptr::from_ref(&$storage).cast_mut().cast(), + }, + }); + }; + }; +} + +/// Read an atomic parameter without borrowing storage that a sysfs write may +/// change concurrently. The module-parameter core supplies a PAGE_SIZE buffer. +/// +/// # Safety +/// +/// `buffer` must be the module-parameter core's writable output buffer and +/// `kp` must be an entry declared by `m3_param!` with AtomicU64 storage. +unsafe extern "C" fn get_atomic_param( + buffer: *mut c_char, + kp: *const kernel::bindings::kernel_param, +) -> c_int { + use core::fmt::Write; + + // SAFETY: The entry points to static AtomicU64 storage by contract. Loads + // and stores are atomic; module-parameter serialization alone would not + // protect the scheduler's concurrent reads. + let value = unsafe { &*((*kp).__bindgen_anon_1.arg.cast::()) } + .load(Ordering::Relaxed); + // SAFETY: 32 bytes fit in the core's PAGE_SIZE output buffer and hold any + // decimal u64 plus its NUL. The core appends the sysfs newline itself. + let bytes = unsafe { core::slice::from_raw_parts_mut(buffer.cast::(), 32) }; + let mut output = kernel::str::Formatter::new(bytes); + if write!(output, "{}\0", value).is_err() { + return EINVAL.to_errno(); + } + (output.bytes_written() - 1) as c_int +} + +fn parse_render_batch_override(text: &str) -> Option { + parse_u64(text).filter(|value| *value <= crate::m3_pass_layout::SLOTS as u64) +} + +/// Runtime tuning is opt-in: zero keeps the existing boot parameter and its +/// clamping semantics. Only root may write a bounded 1..=SLOTS override; an +/// invalid write leaves the previous value intact. Compute batching is separate. +static M3_RENDER_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_render_batch_override", M3_RENDER_BATCH_OVERRIDE, + parse_render_batch_override, 0o644, Some(get_atomic_param)); + +fn parse_render_stamp_coalesce(text:&str)->Option { + parse_u64(text).filter(|value| *value<=1) +} + +/// Root-only T6030 render stamp-flush coalescing; zero retains per-pass flushes. +/// Each publication snapshots the setting before encoding any pass. +static M3_RENDER_STAMP_COALESCE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_render_stamp_coalesce", M3_RENDER_STAMP_COALESCE, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn render_stamp_coalesce()->bool { + M3_RENDER_STAMP_COALESCE.load(Ordering::Relaxed)!=0 +} + +/// T6030 cross-engine publications retain GPU ordering with firmware barriers. +/// Zero preserves the existing host retirement between engine groups. +static M3_MIXED_BATCH: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_mixed_batch", M3_MIXED_BATCH, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn mixed_batch()->bool { + M3_MIXED_BATCH.load(Ordering::Relaxed)!=0 +} + +/// Boot-only T6030 power-state sampling while GPU stamps are outstanding. +/// Zero preserves the completion path without additional register reads. +static M3_ACTIVE_POWER: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_active_power", M3_ACTIVE_POWER, parse_render_stamp_coalesce); +pub(crate) fn active_power()->bool { + M3_ACTIVE_POWER.load(Ordering::Relaxed)==1 +} + +/// Boot-only T6030 Globals debug-word experiment. Zero preserves the runtime word. +static M3_GLOBALS_DEBUG_OFF: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_globals_debug_off",M3_GLOBALS_DEBUG_OFF,parse_render_stamp_coalesce); +pub(crate) fn globals_debug_off()->bool {M3_GLOBALS_DEBUG_OFF.load(Ordering::Relaxed)==1} + +/// T6030-only request while work is pending or active. Zero leaves firmware +/// performance selection unchanged; three requests the top published state. +static M3_CONSISTENT_PERF: AtomicU64 = AtomicU64::new(0); +fn parse_consistent_perf(text: &str)->Option { + parse_u64(text).filter(|v| matches!(*v,0|3)) +} +m3_param!("m3_consistent_perf",M3_CONSISTENT_PERF,parse_consistent_perf,0o644,Some(get_atomic_param)); +pub(crate) fn consistent_perf()->u64 {M3_CONSISTENT_PERF.load(Ordering::Relaxed)} + +/// One preserves internal render measurements. Application timestamp requests +/// remain recorded when zero omits measurements on T6030. +static M3_RENDER_INTERNAL_TIMESTAMPS: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_render_internal_timestamps", M3_RENDER_INTERNAL_TIMESTAMPS, + parse_render_stamp_coalesce, 0o644, Some(get_atomic_param)); +pub(crate) fn render_internal_timestamps()->bool { + M3_RENDER_INTERNAL_TIMESTAMPS.load(Ordering::Relaxed)!=0 +} + +/// Snapshot once per batch. A live change affects later publications, never the +/// storage, ordering or retirement checks of work already on the firmware queues. +pub(crate) fn render_batch_size() -> usize { + let override_size = M3_RENDER_BATCH_OVERRIDE.load(Ordering::Relaxed); + let size = if override_size == 0 { + *crate::module_parameters::m3_render_batch_size.value() as usize + } else { + override_size as usize + }; + size.clamp(1, crate::m3_pass_layout::SLOTS) +} + +fn parse_compute_batch_override(text: &str) -> Option { + parse_u64(text).filter(|value| *value <= crate::m3_compute_storage::SLOTS as u64) +} + +/// Independently tune the existing adjacent-compute path. Zero preserves the +/// boot parameter; invalid writes do not replace the current value. Storage and +/// engine/VM retirement boundaries are unchanged. +static M3_COMPUTE_BATCH_OVERRIDE: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE, + parse_compute_batch_override, 0o644, Some(get_atomic_param)); + +/// Snapshot once per packet, so a live write never changes an in-flight batch. +pub(crate) fn compute_batch_size() -> usize { + let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed); + let size = if override_size == 0 { + *crate::module_parameters::m3_compute_batch_size.value() as usize + } else { + override_size as usize + }; + size.clamp(1, crate::m3_compute_storage::SLOTS) +} + +/// `asahi.g15_debug`: G15 bring-up bits, see [`G15Debug`]. +static G15_DEBUG: AtomicU64 = AtomicU64::new(0); +m3_param!("g15_debug", G15_DEBUG, parse_u64); + +/// Bits of `asahi.g15_debug`, the bring-up switches of the G15 manager backend. +/// +/// The bit numbers are unchanged from their earlier `asahi.debug_flags` positions. Bits 49 +/// and 50 are no longer switches: the HwDataB power-management flags they enabled are always +/// set on G15. +#[derive(Copy, Clone)] +#[repr(u32)] +pub(crate) enum G15Debug { + /// Debug switch: issue the Fender kick (0x11, then 0x10) before starting the ASC. The + /// G15 firmware boots and runs jobs without it. + FenderKick = 41, + /// Register the DRM device (card/render nodes) for the manager backend, like + /// `asahi.m3_expose=1`. Off by default so compositors and system Mesa never see a GPU that + /// cannot run userspace jobs yet; the firmware bring-up runs either way. + ExposeDrm = 42, + /// Turn the firmware log on (RuntimePointers+0x230 = 1) before MSG_INIT. The firmware + /// writes no log records while it is 0. + FwLog = 43, + /// A/B: ring a DEVCTRL (0x83|0x11) and a KICKFW (0x83|0x10) doorbell immediately after + /// MSG_INIT, before the firmware has registered its EP 0x21 handler. + LegacyDoorbells = 44, + /// Skip the post-init DeviceControl liveness probe (no doorbell at all after MSG_INIT). + NoDevctlProbe = 45, + /// Map the gfx-data carveout (firmware data) read-only and log the init progress words, + /// thread handles and crashlog fill at every health sample. + GfxDataPeek = 46, + /// After firmware init, submit one empty compute job and poll its stamp for 2 seconds. + /// Never fails probe and never blocks boot past that poll. + SelfTest = 47, + /// With [`G15Debug::SelfTest`]: first run a firmware-only copy of the job (cmd+0x878 = 1, + /// StartCL skip path, no hardware kick). The real job runs only if that one completes. + SelfTestSkipFirst = 51, + /// Runtime backend: stop the probe right before the GPU coprocessor would be started. The + /// firmware never runs; everything before that point (admission, firmware identity, + /// InitData generation, register identity checks) still does. + StopBeforeAsc = 53, + /// A/B for the manager backend: publish the performance states as the two voltage-sorted + /// tables (with their device-tree index maps) instead of one table in device-tree order. + ManagerSplitPstates = 54, + /// A/B for the manager backend: write the power-controller block's PPM words at +0xc8..+0xd8 + /// in the order the runtime backend's InitData uses (0, enable, target power, kp, ki*dt), + /// and the Globals power-interface targets and performance-state cap, instead of the + /// manager's validated values. + ManagerReferencePpm = 55, + /// Debug switch for the runtime backend: when a job fails and the firmware reports that it has + /// halted, clear the halted flag and set resume, then log for about half a second what the + /// firmware, the engines and the failed job do. The GPU stays marked failed either way. + M3ResumeAfterFault = 56, +} + +/// Returns whether a `asahi.g15_debug` bit is set. +pub(crate) fn g15_debug(bit: G15Debug) -> bool { + G15_DEBUG.load(Ordering::Relaxed) & (1u64 << (bit as u32)) != 0 +} + +/// The raw `asahi.g15_debug` value, for logging. +pub(crate) fn g15_debug_mask() -> u64 { + G15_DEBUG.load(Ordering::Relaxed) +} + +/// `asahi.m3_backend` values. +const BACKEND_AUTO: u64 = 0; +const BACKEND_RUNTIME: u64 = 1; +const BACKEND_MANAGER: u64 = 2; +const BACKEND_OFF: u64 = 3; + +fn parse_backend(text: &str) -> Option { + match text.trim() { + "auto" | "0" => Some(BACKEND_AUTO), + "runtime" | "1" => Some(BACKEND_RUNTIME), + "manager" | "2" => Some(BACKEND_MANAGER), + "off" | "3" => Some(BACKEND_OFF), + _ => None, + } +} + +/// `asahi.m3_backend=auto|runtime|manager|off`: which T6030 GPU backend probe starts. +static M3_BACKEND: AtomicU64 = AtomicU64::new(BACKEND_AUTO); +m3_param!("m3_backend", M3_BACKEND, parse_backend); + +/// The T6030 GPU backend to start. +pub(crate) enum T6030Backend { + /// The serialized M3 runtime (m3_drm / m3_runtime). + Runtime, + /// The G15 GpuManager backend (g15_probe). + Manager, + /// No backend: the probe stops without touching the GPU. + Off, +} + +/// `asahi.m3_initdata` values. +const INITDATA_AUTO: u64 = 0; +const INITDATA_ADT: u64 = 2; + +fn parse_initdata(text: &str) -> Option { + match text.trim() { + "auto" | "0" => Some(INITDATA_AUTO), + "adt" | "2" => Some(INITDATA_ADT), + _ => None, + } +} + +/// `asahi.m3_initdata=auto|adt`: the runtime backend's InitData contents are generated from the +/// device tree either way; the value is logged. +static M3_INITDATA: AtomicU64 = AtomicU64::new(INITDATA_AUTO); +m3_param!("m3_initdata", M3_INITDATA, parse_initdata); + +/// Requested source of the runtime backend's InitData contents. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum InitDataParam { + /// Not given: contents generated from the device tree. + Auto, + /// Contents generated from the device tree. + Adt, +} + +/// The `asahi.m3_initdata` setting. +pub(crate) fn initdata_param() -> InitDataParam { + match M3_INITDATA.load(Ordering::Relaxed) { + INITDATA_ADT => InitDataParam::Adt, + _ => InitDataParam::Auto, + } +} + +/// Default of `asahi.m3_max_pstate` for device-tree InitData and the G15 manager backend: the +/// fifth performance state (1056 MHz on J516S). +pub(crate) const MAX_PSTATE_DEFAULT: u64 = 5; +/// A performance-state parameter that is not given. +const PSTATE_UNSET: u64 = u64::MAX; + +/// `asahi.m3_max_pstate`: the highest GPU performance state the driver lets the firmware use, +/// as an index into the voltage-sorted performance-state table (1 = lowest, 8 = highest on +/// T6030; the G15 manager backend translates it to its device-tree ordered table). It is +/// clamped to the device tree's highest performance state. Unset means +/// [`MAX_PSTATE_DEFAULT`], except with the runtime backend while its thermal limit is on or +/// holding, which keeps the whole table. +static M3_MAX_PSTATE: AtomicU64 = AtomicU64::new(PSTATE_UNSET); +m3_param!("m3_max_pstate", M3_MAX_PSTATE, parse_u64); + +/// `asahi.m3_boot_pstate`: the performance state the runtime backend requests at boot, clamped +/// to the cap. Unset means the cap. +static M3_BOOT_PSTATE: AtomicU64 = AtomicU64::new(PSTATE_UNSET); +m3_param!("m3_boot_pstate", M3_BOOT_PSTATE, parse_u64); + +/// The `asahi.m3_max_pstate` setting, if given, before clamping to the performance-state table. +pub(crate) fn max_pstate_param() -> Option { + Some(M3_MAX_PSTATE.load(Ordering::Relaxed)).filter(|v| *v != PSTATE_UNSET) +} + +/// The `asahi.m3_boot_pstate` setting, if given, before clamping. +pub(crate) fn boot_pstate_param() -> Option { + Some(M3_BOOT_PSTATE.load(Ordering::Relaxed)).filter(|v| *v != PSTATE_UNSET) +} + +/// `asahi.m3_timeout_nohang`: 1 (default) makes a scheduler timeout of a runtime-backend job +/// report "no hang", because the runtime completes every packet itself and enforces its own +/// per-batch completion limit. 0 keeps the previous handling: mark the GPU failed, fail the +/// packet again and report the device as gone. +static M3_TIMEOUT_NOHANG: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_timeout_nohang", M3_TIMEOUT_NOHANG, parse_u64); + +/// Whether `asahi.m3_timeout_nohang` is set (the default). +pub(crate) fn timeout_nohang() -> bool { + M3_TIMEOUT_NOHANG.load(Ordering::Relaxed) != 0 +} + +/// `asahi.m3_unlocked_wait`: 1 (default) releases the runtime lock while the runtime backend +/// waits for a batch to complete, so VM creation, timestamp mapping and the event worker are +/// not held up by a running job. 0 holds the lock from preparation to retirement, as before. +static M3_UNLOCKED_WAIT: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_unlocked_wait", M3_UNLOCKED_WAIT, parse_u64); + +/// Whether `asahi.m3_unlocked_wait` is set (the default). +pub(crate) fn unlocked_wait() -> bool { + M3_UNLOCKED_WAIT.load(Ordering::Relaxed) != 0 +} + +/// `asahi.m3_ut_engagement=0|1`: Globals +0x7d0, the gate of the firmware's frequency-feedback +/// cap. Unset: the SoC's default ([`crate::m3_soc::Soc::ut_engagement`]). +static M3_UT_ENGAGEMENT: AtomicU64 = AtomicU64::new(UT_ENGAGEMENT_UNSET); +const UT_ENGAGEMENT_UNSET: u64 = u64::MAX; +fn parse_ut_engagement(text: &str) -> Option { + parse_u64(text).filter(|v| *v <= 1) +} +m3_param!("m3_ut_engagement", M3_UT_ENGAGEMENT, parse_ut_engagement); + +/// The frequency-feedback gate for `soc`, and whether `asahi.m3_ut_engagement` set it. +pub(crate) fn ut_engagement(soc: &crate::m3_soc::Soc) -> (u32, bool) { + match M3_UT_ENGAGEMENT.load(Ordering::Relaxed) { + UT_ENGAGEMENT_UNSET => (soc.ut_engagement, false), + v => (v as u32, true), + } +} + +/// `asahi.m3_pipeline_depth`: writable at runtime. How many render batches may be in flight on +/// the firmware's render queues. Zero selects the SoC default: 2 on the explicitly enabled +/// experimental T8122 runtime, 1 on T6030. Depth 1 publishes only after the previous batch has +/// fully retired. Larger values publish the next render batch of the same VM while earlier +/// ones still run, in disjoint pass slots, ordered by the existing cross-pass barriers. +static M3_PIPELINE_DEPTH: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_pipeline_depth", M3_PIPELINE_DEPTH, parse_pipeline_depth, 0o644, Some(get_atomic_param)); + +fn parse_pipeline_depth(text: &str) -> Option { + parse_u64(text).filter(|v| *v<=crate::m3_pass_layout::SLOTS as u64) +} + +/// Effective depth; changing it never releases already published pass slots. +pub(crate) fn pipeline_depth(soc: &crate::m3_soc::Soc) -> usize { + match M3_PIPELINE_DEPTH.load(Ordering::Relaxed) { + 0 => crate::m3_runtime::policy::default_depth(crate::t8122_start::is_t8122(soc)), + v => v.clamp(1,crate::m3_pass_layout::SLOTS as u64) as usize, + } +} + +/// `asahi.m3_render_batch_budget_us`: writable at runtime. A render batch takes in later packets +/// of its VM only while the VM's measured GPU time per pass, times the passes already in the +/// batch, stays below this budget (4000 µs on experimental T8122, disabled on Pro). +/// Coalescing a client's heavy frames +/// otherwise makes every other client's work, the compositor's included, wait for all of them. +/// 0 disables the budget; u64::MAX restores the SoC default. +static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); +m3_param!("m3_render_batch_budget_us", M3_RENDER_BATCH_BUDGET_US, parse_u64, 0o644, Some(get_atomic_param)); + +/// The render batch GPU-time budget in ns, or `None` without one. +pub(crate) fn render_batch_budget_ns(soc: &crate::m3_soc::Soc) -> Option { + let value=M3_RENDER_BATCH_BUDGET_US.load(Ordering::Relaxed); + let us=if value==u64::MAX {crate::m3_runtime::policy::default_budget_us(crate::t8122_start::is_t8122(soc))} else {value}; + match us { + 0 => None, + us => Some(us.saturating_mul(1000)), + } +} + +/// Bits of `asahi.m3_retire_mmio`. +pub(crate) const RETIRE_MMIO_BUSY: u64 = 1 << 0; +pub(crate) const RETIRE_MMIO_FAULTS: u64 = 1 << 1; +pub(crate) const RETIRE_MMIO_PSTATE: u64 = 1 << 2; +const RETIRE_MMIO_UNSET: u64 = u64::MAX; + +fn parse_retire_mmio(text: &str) -> Option { + parse_u64(text).filter(|v| *v <= 7) +} + +/// `asahi.m3_retire_mmio`, writable at runtime: the GPU registers the host reads after every +/// job before retiring it. Bit 0 reads the engine-busy registers (SGX+0xc020/0xc120), bit 1 +/// selects and reads each cluster's fault bank (0xd800/0xd8c0), bit 2 reads the performance +/// state (0xe01000). Unset: the SoC's default ([`crate::m3_soc::Soc::retire_mmio`]). +static M3_RETIRE_MMIO: AtomicU64 = AtomicU64::new(RETIRE_MMIO_UNSET); +m3_param!("m3_retire_mmio", M3_RETIRE_MMIO, parse_retire_mmio, 0o644, Some(get_atomic_param)); + +/// The `asahi.m3_retire_mmio` mask for `soc`. +pub(crate) fn retire_mmio(soc: &crate::m3_soc::Soc) -> u64 { + match M3_RETIRE_MMIO.load(Ordering::Relaxed) { + RETIRE_MMIO_UNSET => soc.retire_mmio, + mask => mask, + } +} + +/// `asahi.m3_thermal` values. +const THERMAL_OFF: u64 = 0; +const THERMAL_HOLD: u64 = 1; +const THERMAL_ON: u64 = 2; +const THERMAL_UNSET: u64 = u64::MAX; + +fn parse_thermal(text: &str) -> Option { + match text.trim() { + "off" | "0" => Some(THERMAL_OFF), + "hold" => Some(THERMAL_HOLD), + "on" | "1" => Some(THERMAL_ON), + _ => None, + } +} + +static M3_THERMAL: AtomicU64 = AtomicU64::new(THERMAL_UNSET); +m3_param!("m3_thermal", M3_THERMAL, parse_thermal); + +/// How the runtime backend limits the GPU performance state by temperature. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub(crate) enum ThermalMode { + /// Today's fixed cap: the firmware is given the cap as its highest state, and nothing + /// changes it at runtime. + Off, + /// The firmware is given the whole table as its highest state, and the runtime cap words + /// hold the fixed cap for the whole boot. The same performance as `Off`, if the firmware + /// follows the runtime cap: this checks that it does. + Hold, + /// As `Hold`, and the runtime cap follows the SoC die temperature, up to the whole table + /// while the die is cool. + On, +} + +/// The `asahi.m3_thermal` setting, if given. +pub(crate) fn thermal_param() -> Option { + match M3_THERMAL.load(Ordering::Relaxed) { + THERMAL_OFF => Some(ThermalMode::Off), + THERMAL_HOLD => Some(ThermalMode::Hold), + THERMAL_ON => Some(ThermalMode::On), + _ => None, + } +} + +/// Default and bounds of `asahi.m3_thermal_hot`, in degrees Celsius. +pub(crate) const THERMAL_HOT_DEFAULT: u64 = 80; +pub(crate) const THERMAL_HOT_MIN: u64 = 40; +pub(crate) const THERMAL_HOT_MAX: u64 = 85; + +/// `asahi.m3_thermal_hot`: the SoC die temperature (degrees Celsius) at or above which the thermal +/// limit lowers the runtime cap, clamped to [`THERMAL_HOT_MIN`]..=[`THERMAL_HOT_MAX`]. +static M3_THERMAL_HOT: AtomicU64 = AtomicU64::new(THERMAL_HOT_DEFAULT); +m3_param!("m3_thermal_hot", M3_THERMAL_HOT, parse_u64); + +/// The `asahi.m3_thermal_hot` setting and whether it was clamped. +pub(crate) fn thermal_hot_param() -> (u64, bool) { + let value = M3_THERMAL_HOT.load(Ordering::Relaxed); + let clamped = value.clamp(THERMAL_HOT_MIN, THERMAL_HOT_MAX); + (clamped, clamped != value) +} + +/// Whether to start the runtime backend on a T8122 (`asahi.m3_backend`), and say why not. +/// +/// The runtime starts only on the boards the T8122 table allows (the M3 MacBook Airs), with +/// `auto` and `runtime` alike: it has not run on any T8122, so nothing forces it onto another +/// board. The G15 manager backend is T6030-only. +pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { + let dev = pdev.as_ref(); + match M3_BACKEND.load(Ordering::Relaxed) { + BACKEND_OFF => { + dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); + false + } + BACKEND_MANAGER => { + dev_info!( + dev, + "M3: asahi.m3_backend=manager: the G15 manager backend is T6030-only; no GPU backend started\n" + ); + false + } + _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T8122) => true, + _ => { + dev_info!( + dev, + "M3: no GPU backend on this T8122 board: the runtime backend may start only on the M3 MacBook Airs (J613, J615)\n" + ); + false + } + } +} + +/// Select the T6030 GPU backend from `asahi.m3_backend`, and say which one and why. +/// +/// `auto` starts the runtime backend on the boards it is validated on (the M3 Pro MacBook +/// Pros). On other T6030 boards it starts nothing until the runtime is validated there; +/// `asahi.m3_backend=runtime` starts it anyway. +pub(crate) fn t6030_backend(pdev: &platform::Device) -> T6030Backend { + let dev = pdev.as_ref(); + match M3_BACKEND.load(Ordering::Relaxed) { + BACKEND_RUNTIME => { + dev_info!(dev, "M3: asahi.m3_backend=runtime\n"); + T6030Backend::Runtime + } + BACKEND_MANAGER => { + dev_info!(dev, "M3: asahi.m3_backend=manager: starting the G15 manager backend\n"); + T6030Backend::Manager + } + BACKEND_OFF => { + dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); + T6030Backend::Off + } + _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T6030) => T6030Backend::Runtime, + _ => { + dev_info!( + dev, + "M3: asahi.m3_backend=auto: no GPU backend on this board: the runtime backend is not validated here yet (asahi.m3_backend=runtime starts it, asahi.m3_backend=manager starts the G15 manager backend)\n" + ); + T6030Backend::Off + } + } +} + +fn parse_t8122_number(text: &str) -> Option { + Some(crate::t8122_knobs::parse_number(text)) +} + +fn parse_t8122_fender(text: &str) -> Option { + Some(crate::t8122_knobs::parse_fender(text)) +} + +fn parse_t8122_clkgen(text: &str) -> Option { + Some(crate::t8122_knobs::parse_clkgen(text)) +} + +fn parse_t8122_sgx_setup(text: &str) -> Option { + Some(crate::t8122_knobs::parse_sgx_setup(text)) +} + +// The T8122 start experiment (`t8122_start`). Each parameter is stored as given, unset or with a +// value its parser does not accept (`t8122_knobs`), so that the experiment can refuse rather +// than fall back to a default. +use crate::t8122_knobs::UNSET as T8122_UNSET; + +/// `asahi.t8122_start=1`: start the M3 runtime on a T8122 with the start experiment's values. +/// Only with a boot loader that handed the GPU over. +static T8122_START: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_start", T8122_START, parse_t8122_number); +/// `asahi.t8122_initdata_version=`: the InitData version given to the T8122 firmware. +static T8122_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_initdata_version", T8122_INITDATA_VERSION, parse_t8122_number); +/// `asahi.t8122_fender=0x104000|0x12c000` (`rule`, `adt`): the Fender window size. +static T8122_FENDER: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_fender", T8122_FENDER, parse_t8122_fender); +/// `asahi.t8122_clkgen=e1c|e5c|none`: the GPU clock-generator IO mapping. +static T8122_CLKGEN: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_clkgen", T8122_CLKGEN, parse_t8122_clkgen); +/// `asahi.t8122_sgx_setup=none|t6030`: the SGX write made before the firmware starts. +static T8122_SGX_SETUP: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_sgx_setup", T8122_SGX_SETUP, parse_t8122_sgx_setup); +/// `asahi.t8122_unit_mask_a=`: HwDataB unit mask A (+0x17c0). +static T8122_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_unit_mask_a", T8122_UNIT_MASK_A, parse_t8122_number); +/// `asahi.t8122_unit_mask_b=`: HwDataB unit mask B (+0x17c8). +static T8122_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_unit_mask_b", T8122_UNIT_MASK_B, parse_t8122_number); +/// `asahi.t8122_pstate_cap=N`: the highest performance state the T8122 firmware may use. +static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET); +m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number); + +/// `asahi.t8122_hwdata_418c=0|1`: on a T8122 with the G15 14.8.3 firmware, 1 (the default) +/// writes the 20 words at HwDataA +0x418c ([`crate::t8122_hwdata`]); 0 leaves the +/// range zero, for an A/B test. Readable in sysfs. +static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::t8122_hwdata::DEFAULT); +m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::t8122_hwdata::parse, 0o444, + Some(get_atomic_param)); + +/// Whether `asahi.t8122_hwdata_418c` asks for the HwDataA +0x418c words (the default). +pub(crate) fn t8122_hwdata_418c() -> bool { + crate::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) +} + +/// The T8122 start-experiment parameters of this boot, as given. +pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { + let get = |p: &AtomicU64| p.load(Ordering::Relaxed); + crate::t8122_knobs::Raw { + start: get(&T8122_START), + initdata_version: get(&T8122_INITDATA_VERSION), + fender: get(&T8122_FENDER), + clkgen: get(&T8122_CLKGEN), + sgx_setup: get(&T8122_SGX_SETUP), + unit_mask_a: get(&T8122_UNIT_MASK_A), + unit_mask_b: get(&T8122_UNIT_MASK_B), + pstate_cap: get(&T8122_PSTATE_CAP), + } +} + +fn parse_t6031_number(text: &str) -> Option { + Some(crate::t6031_knobs::parse_number(text)) +} + +fn parse_t6031_fender(text: &str) -> Option { + Some(crate::t6031_knobs::parse_fender(text)) +} + +fn parse_t6031_clkgen(text: &str) -> Option { + Some(crate::t6031_knobs::parse_clkgen(text)) +} + +fn parse_t6031_sgx_setup(text: &str) -> Option { + Some(crate::t6031_knobs::parse_sgx_setup(text)) +} + +fn parse_t6031_mcache(text: &str) -> Option { + Some(crate::t6031_knobs::parse_mcache(text)) +} + +fn parse_t6031_aic_swint(text: &str) -> Option { + Some(crate::t6031_knobs::parse_aic_swint(text)) +} + +fn parse_t6031_slot7(text: &str) -> Option { + Some(crate::t6031_knobs::parse_slot7(text)) +} + +fn parse_t6031_gifaf(text: &str) -> Option { + Some(crate::t6031_knobs::parse_gifaf(text)) +} + +fn parse_t6031_slot(text: &str) -> Option { + Some(crate::t6031_knobs::parse_slot(text)) +} + +fn parse_t6031_ane(text: &str) -> Option { + Some(crate::t6031_knobs::parse_ane(text)) +} + +fn parse_t6031_mtr(text: &str) -> Option { + Some(crate::t6031_knobs::parse_mtr(text)) +} + +fn parse_t6031_fw_words(text: &str) -> Option { + Some(crate::t6031_knobs::parse_fw_words(text)) +} + +fn parse_t6031_hwdata_object(text: &str) -> Option { + Some(crate::t6031_knobs::parse_hwdata_object(text)) +} + +fn parse_t6031_rev_id(text: &str) -> Option { + Some(crate::t6031_knobs::parse_rev_id(text)) +} + +fn parse_t6031_tristate(text: &str) -> Option { + Some(crate::t6031_knobs::parse_tristate(text)) +} + +fn parse_t6031_tag(text: &str) -> Option { + Some(crate::t6031_knobs::parse_tag(text)) +} + +// The T6031 start (`t6031_start`). Each parameter is stored as given, unset or with a value its +// parser does not accept (`t6031_knobs`), so the start can refuse rather than fall back. +use crate::t6031_knobs::UNSET as T6031_UNSET; + +/// `asahi.t6031_start=1`: one attempt to start the M3 runtime on a T6031. Without it the probe +/// is the same refusal as before. +static T6031_START: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_start", T6031_START, parse_t6031_number); +/// `asahi.t6031_image_hash`: the first 8 bytes of the loaded text SHA-256, big-endian. +static T6031_IMAGE_HASH: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_image_hash", T6031_IMAGE_HASH, parse_t6031_number); +/// `asahi.t6031_tag=search` or a text offset of the boot-entropy tag. +static T6031_TAG: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_tag", T6031_TAG, parse_t6031_tag); +/// `asahi.t6031_initdata_version=`: the InitData version given to the T6031 firmware. +static T6031_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_initdata_version", T6031_INITDATA_VERSION, parse_t6031_number); +/// `asahi.t6031_pstate_cap=N`: the highest performance state the T6031 firmware may use. +static T6031_PSTATE_CAP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_pstate_cap", T6031_PSTATE_CAP, parse_t6031_number); +/// `asahi.t6031_fender=rule|adt`: the Fender window size. +static T6031_FENDER: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_fender", T6031_FENDER, parse_t6031_fender); +/// `asahi.t6031_clkgen=e5c|e1c|none`: the GPU clock-generator IO mapping. +static T6031_CLKGEN: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_clkgen", T6031_CLKGEN, parse_t6031_clkgen); +/// `asahi.t6031_sgx_setup=t6030|none`: the SGX write made before the firmware starts. +static T6031_SGX_SETUP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_sgx_setup", T6031_SGX_SETUP, parse_t6031_sgx_setup); +/// `asahi.t6031_mcache=none|x2|x4|x8`. +static T6031_MCACHE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mcache", T6031_MCACHE, parse_t6031_mcache); +/// `asahi.t6031_aic_swint=page|reg`. +static T6031_AIC_SWINT: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_aic_swint", T6031_AIC_SWINT, parse_t6031_aic_swint); +/// `asahi.t6031_slot7=none|rule`. +static T6031_SLOT7: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot7", T6031_SLOT7, parse_t6031_slot7); +/// `asahi.t6031_gifaf=none|fender`. +static T6031_GIFAF: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_gifaf", T6031_GIFAF, parse_t6031_gifaf); +/// `asahi.t6031_slot21=none` or a physical address. +static T6031_SLOT21: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot21", T6031_SLOT21, parse_t6031_slot); +/// `asahi.t6031_slot24=none` or a physical address. +static T6031_SLOT24: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot24", T6031_SLOT24, parse_t6031_slot); +/// `asahi.t6031_slot28=none` or a physical address. +static T6031_SLOT28: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_slot28", T6031_SLOT28, parse_t6031_slot); +/// `asahi.t6031_ane=none|t6030|t8122` or a physical address. +static T6031_ANE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_ane", T6031_ANE, parse_t6031_ane); +/// `asahi.t6031_io_drop`: a mask of default slots to leave out. +static T6031_IO_DROP: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_io_drop", T6031_IO_DROP, parse_t6031_number); +/// `asahi.t6031_mtr=t6030|t8122|adt|none`. +static T6031_MTR: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr", T6031_MTR, parse_t6031_mtr); +/// `asahi.t6031_mtr_fast_die`: overrides the fast-die MTR mask. +static T6031_MTR_FAST_DIE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr_fast_die", T6031_MTR_FAST_DIE, parse_t6031_number); +/// `asahi.t6031_mtr_alarm`: overrides the alarm MTR mask. +static T6031_MTR_ALARM: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_mtr_alarm", T6031_MTR_ALARM, parse_t6031_number); +/// `asahi.t6031_unit_mask_a=`: HwDataB unit mask A (+0x17c0). +static T6031_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_unit_mask_a", T6031_UNIT_MASK_A, parse_t6031_number); +/// `asahi.t6031_unit_mask_b=`: HwDataB unit mask B (+0x17c8). +static T6031_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_unit_mask_b", T6031_UNIT_MASK_B, parse_t6031_number); +/// `asahi.t6031_hwb_454`: HwDataB +0xa2c. +static T6031_HWB_454: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_454", T6031_HWB_454, parse_t6031_number); +/// `asahi.t6031_hwb_b20`: HwDataB +0xb20, the core slots. +static T6031_HWB_B20: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_b20", T6031_HWB_B20, parse_t6031_number); +/// `asahi.t6031_hwb_17b8`: HwDataB +0x17b8. +static T6031_HWB_17B8: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_17b8", T6031_HWB_17B8, parse_t6031_number); +/// `asahi.t6031_hwb_1818`: HwDataB +0x1818. +static T6031_HWB_1818: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwb_1818", T6031_HWB_1818, parse_t6031_number); +/// `asahi.t6031_fw_words=none|t8122`. +static T6031_FW_WORDS: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_fw_words", T6031_FW_WORDS, parse_t6031_fw_words); +/// `asahi.t6031_hwdata_object=aligned|fixed`. +static T6031_HWDATA_OBJECT: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_hwdata_object", T6031_HWDATA_OBJECT, parse_t6031_hwdata_object); +/// `asahi.t6031_gpu_core`: must be the table's G15C core type, 24. +static T6031_GPU_CORE: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_gpu_core", T6031_GPU_CORE, parse_t6031_number); +/// `asahi.t6031_rev_id=auto` or 1 to 6. +static T6031_REV_ID: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_rev_id", T6031_REV_ID, parse_t6031_rev_id); +/// `asahi.t6031_csafr=auto|on|off`. +static T6031_CSAFR: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_csafr", T6031_CSAFR, parse_t6031_tristate); +/// `asahi.t6031_power_cap_mw`: the power target while the boot loader's model is a stand-in. +static T6031_POWER_CAP_MW: AtomicU64 = AtomicU64::new(T6031_UNSET); +m3_param!("t6031_power_cap_mw", T6031_POWER_CAP_MW, parse_t6031_number); + +/// The T6031 start parameters of this boot, as given. The image UUID has no module parameter. +pub(crate) fn t6031_params() -> crate::t6031_knobs::Raw { + let get = |p: &AtomicU64| p.load(Ordering::Relaxed); + crate::t6031_knobs::Raw { + start: get(&T6031_START), + image_uuid: crate::t6031_knobs::UuidParam::Unset, + image_hash: get(&T6031_IMAGE_HASH), + tag: get(&T6031_TAG), + initdata_version: get(&T6031_INITDATA_VERSION), + pstate_cap: get(&T6031_PSTATE_CAP), + fender: get(&T6031_FENDER), + clkgen: get(&T6031_CLKGEN), + sgx_setup: get(&T6031_SGX_SETUP), + mcache: get(&T6031_MCACHE), + aic_swint: get(&T6031_AIC_SWINT), + slot7: get(&T6031_SLOT7), + gifaf: get(&T6031_GIFAF), + slot21: get(&T6031_SLOT21), + slot24: get(&T6031_SLOT24), + slot28: get(&T6031_SLOT28), + ane: get(&T6031_ANE), + io_drop: get(&T6031_IO_DROP), + mtr: get(&T6031_MTR), + mtr_fast_die: get(&T6031_MTR_FAST_DIE), + mtr_alarm: get(&T6031_MTR_ALARM), + unit_mask_a: get(&T6031_UNIT_MASK_A), + unit_mask_b: get(&T6031_UNIT_MASK_B), + hwb_454: get(&T6031_HWB_454), + hwb_b20: get(&T6031_HWB_B20), + hwb_17b8: get(&T6031_HWB_17B8), + hwb_1818: get(&T6031_HWB_1818), + fw_words: get(&T6031_FW_WORDS), + hwdata_object: get(&T6031_HWDATA_OBJECT), + gpu_core: get(&T6031_GPU_CORE), + rev_id: get(&T6031_REV_ID), + csafr: get(&T6031_CSAFR), + power_cap_mw: get(&T6031_POWER_CAP_MW), + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs new file mode 100644 index 00000000000000..273d2701c3f143 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Private storage for one retired/unpublished G15 render pass. A Pass retains +//! all its buffers and mappings until Render retires the shared queues. +use kernel::prelude::*; +use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_pass_layout as layout}; +use layout::{Field,Space}; + +pub(crate) struct Pass {buffers:KVec} +impl Pass { + pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + slot:usize,clusters:u32,aliases:&mut KVec)->Result { + let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; + for field in layout::FIELDS { + let a=layout::board_allocation(slot,field,clusters).map_err(|_|EINVAL)?; + let mut b=match a.space { + Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, + a.size,prot::PROT_GPU_FW_SHARED_RW,prot::Prot::from_pte(a.access.pte()))?, + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(), + a.gpu_alias.map(|address|(uat.kernel_lower_vm(),address)),Some(a.address), + a.size,prot::Prot::from_pte(a.access.pte()),prot::PROT_GPU_SHARED_RW)?, + }; + if a.gpu_alias.is_some() {aliases.push(b.map_gpu_view(vm)?,GFP_KERNEL)?;} + buffers.push(b,GFP_KERNEL)?; + } + Ok(Self{buffers}) + } + pub(crate) fn get(&self,field:Field)->&Buffer {&self.buffers[field as usize]} + pub(crate) fn get_mut(&mut self,field:Field)->&mut Buffer {&mut self.buffers[field as usize]} + /// Caller must prove retirement, and must not clear a prepared sibling. + /// Dependency commands are rewritten completely by the sync encoder. + pub(crate) fn reset(&mut self,geometry:&crate::agx_render::Geometry)->Result { + // Tiler arenas grow to retain the largest prior target. The firmware + // descriptors use the current geometry's strides and layer count, so + // only that prefix is addressable by this pass. Clear it on every use, + // including when growing again after a smaller pass; the unused tail + // may still contain retired data and is never assumed to be zero. + for field in &layout::FIELDS[..layout::RESET_COUNT] { + let bytes=match field { + Field::Tpc=>usize::try_from(geometry.tpc_bytes).map_err(|_|EOVERFLOW)?, + Field::Tilemap=>usize::try_from(geometry.tilemap_bytes).map_err(|_|EOVERFLOW)?, + _=>self.get(*field).size(), + }; + self.get_mut(*field).fill_range(0,bytes,0)?; + } + Ok(()) + } +} +/// Cached client views retain their resource identity across switches. A pass +/// field cannot accidentally name a shared allocator or another pass's field. +#[derive(Clone,Copy)] +pub(crate) enum ViewOwner {Shared(usize),Pass{slot:usize,field:Field}} diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs new file mode 100644 index 00000000000000..979c1a37bb202b --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Construct a firmware-owned UMA allocator once from its graph's allocations. +//! GPU mappings and counters outlive this initialization and remain owned by +//! Render/Compute. Callers must never invoke this on a live or replayed pool. +use kernel::prelude::*; +use crate::{m3_memory::Buffer, m3_pool_layout as wire, m3_queue_layout::FirmwareVa}; +pub(crate) struct Owners { + pub(crate) backing: core::ops::Range, + pub(crate) pages: usize, + pub(crate) table: usize, + pub(crate) manager: usize, + pub(crate) counter: usize, + pub(crate) table_index: u8, +} +impl Owners { + pub(crate) fn initialize(&self, objects: &mut [Buffer]) -> Result { + let buffers = objects.get(self.backing.clone()).ok_or(EINVAL)?; + let mut backings = KVec::with_capacity(buffers.len(), GFP_KERNEL)?; + for buffer in buffers { + backings.push(wire::GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?, GFP_KERNEL)?; + } + let pages = objects.get(self.pages).ok_or(EINVAL)?; + let pages = wire::GpuRegion::new(pages.gpu_va()?, pages.size()).map_err(|_| EINVAL)?; + let table = objects.get(self.table).ok_or(EINVAL)?; + let table = wire::GpuRegion::new(table.gpu_va()?, table.size()).map_err(|_| EINVAL)?; + let counter = FirmwareVa::new(objects.get(self.counter).ok_or(EINVAL)?.va()).map_err(|_| EINVAL)?; + if objects.get(self.manager).ok_or(EINVAL)?.size() < wire::MANAGER_SIZE { return Err(EINVAL); } + let pool = wire::Pool::new(&backings, pages, table, counter, self.table_index).map_err(|_| EINVAL)?; + let mut manager = [0; wire::MANAGER_SIZE]; + pool.encode(&mut manager).map_err(|_| EINVAL)?; + // Unpopulated entries remain explicitly zero from the fresh DMA owner. + for page in 0..wire::INITIAL_PAGES { + objects[self.pages].u64(page * 8, pool.page(page).map_err(|_| EINVAL)?)?; + } + for entry in 0..backings.len() { + objects[self.table].write(entry * wire::POOL_ENTRY_SIZE, + &pool.table_entry(entry).map_err(|_| EINVAL)?)?; + } + objects[self.manager].write(0, &manager) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs new file mode 100644 index 00000000000000..c2c67f5c3aa5ac --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -0,0 +1,696 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use kernel::prelude::*; +use crate::m3_queue_layout as queue; +use crate::m3_sync_layout as sync; +use crate::m3_state_layout as state; +use crate::m3_scene_layout as scene; +use crate::{m3_pass::{Pass,ViewOwner},m3_pass_layout::{self as pass_layout,Field as P,Space}}; +use crate::{m3_fragment_command as fragment, m3_tiler_command as tiler, m3_render_sequence as sequence, m3_init_layout::Region}; +use crate::{m3_parameter_layout as parameter, m3_pool_layout::GpuRegion}; +use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_shared_layout as s,agx_uapi}; +use crate::m3_timeline as timeline; +use s::{PB_FIRST,PB_GROUPS,PB_BLOCKS_PER_GROUP,PB_BLOCK_SIZE}; +const PB_BLOCKS:u32=(PB_GROUPS*PB_BLOCKS_PER_GROUP) as u32; +pub(crate) const PB_PAGES:u32=PB_BLOCKS*4; +// Serial execution permits retaining each client's views of the owned arenas. +// Bound the cache to four VMs including the active one; cache entries retain +// their ASID lease and all aliases until eviction after exact retirement. +struct CachedViews { + aliases:KVec, + views:KVec<(ViewOwner,mmu::KernelMapping)>, + // Release mappings before returning their ASID lease (Rust field order). + binding:mmu::VmBind, +} +pub(crate) struct Render { + _aliases:KVec, objects:KVec, + passes:KVec, init_bm:Buffer, + // All active mappings above must drop before their ASID binding. + _binding:mmu::VmBind, + stats: Region, sequence_bytes: KVec, tiler_bytes: KVec, fragment_bytes: KVec, + last_progress_ns:i64, + draw:u64, heads:[u16;2], cached_views:KVec, + slot:usize, batch_count:usize, early_count:u32, + /// Measurement packets in the pass most recently appended to its queues. + stage_timestamps:[bool;2], + /// First pass slot of the current batch; its passes use base..base+batch_count mod SLOTS. + base:usize, + checkpoint:(u64,[u16;2],usize,usize,u32,usize), + /// GPU clusters of the board: tiler accelerators (TPC storage) and preemption buffers. + clusters:u32, + /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). + registers:crate::m3_compute_layout::RegisterSet, +} +// Queue allocations belong to Render, not to a pass. Keep their fixed arenas +// and firmware/shared mapping permissions from the qualified allocation map. +// Scratch is GPU-visible firmware storage; it is not the application's VM. +struct QueueObjects { + info: usize, state: usize, ring: usize, scratch: usize, + tiler: bool, uuid: u32, +} +use s::{GPU_CONTEXT,JOB_LIST,BUFFER_MANAGER,PARAMETER_PAGE_LIST, + PARAMETER_BLOCK_LIST,NOTIFIER,BLOCK_CONTROL,MANAGER_MISC}; +const MANAGER_SCENE_COUNT: u32 = 48; +const TA_POOL_COUNTER: usize = 11; +const UMA_POOLS: [crate::m3_pool::Owners; 2] = [ + crate::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, + crate::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, +]; +const FRAGMENT_POOL_COUNTER: usize = 23; +// Retained producer identifier for cross-pass dependencies, G15 / V14_8_3. +const DEPENDENCY_UUID: u32 = 250912; +const RENDER_QUEUES: [QueueObjects; 2] = [ + QueueObjects { info: s::TA_QUEUE, state: s::TA_POINTERS, ring: s::TA_RING, + scratch: s::TA_SCRATCH, tiler: true, uuid: 0x7a0000 }, + QueueObjects { info: s::FRAGMENT_QUEUE, state: s::FRAGMENT_POINTERS, ring: s::FRAGMENT_RING, + scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, +]; +impl Render { + pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, + registers:crate::m3_compute_layout::RegisterSet)->Result { + let binding=uat.bind(vm)?; + let mut objects=KVec::new();let mut aliases=KVec::new(); + for index in 0..s::COUNT { + let a=s::allocation(index).map_err(|_|EINVAL)?; + let protection=prot::Prot::from_pte(a.access.pte()); + let b=match a.space { + Space::ClientGpu=>Buffer::at_prot(dev,uat.kernel_vm(),Some((vm,a.address)),None, + a.size,prot::PROT_GPU_FW_SHARED_RW,protection)?, + Space::Firmware=>Buffer::at_prot(dev,uat.kernel_vm(),None,Some(a.address), + a.size,protection,prot::PROT_GPU_SHARED_RW)?, + }; + objects.push(b,GFP_KERNEL)?; + } + // InitBM is shared, not part of a pass. Preserve its qualified mapping. + let init_bm=Buffer::at_prot(dev,uat.kernel_vm(),None,Some(0xfffffc200062bfe0), + sync::INIT_BM_SIZE,prot::PROT_FW_SHARED_RW,prot::PROT_GPU_SHARED_RW)?; + let mut passes=KVec::with_capacity(pass_layout::SLOTS,GFP_KERNEL)?; + for slot in 0..pass_layout::SLOTS {passes.push(Pass::new(dev,uat,vm,slot,clusters,&mut aliases)?,GFP_KERNEL)?;} + let mut sequence_bytes=KVec::new(); + sequence_bytes.resize(sequence::STORAGE,0u8,GFP_KERNEL)?; + let mut tiler_bytes=KVec::new(); + tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; + let mut fragment_bytes=KVec::new(); + fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; + let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers}; + // Lists and the matching manager are built before InitBM publication. + job.initialize_parameter_buffer()?; + for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } + job.initialize_state()?; + job.initialize_queues()?; + dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); + crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; + Ok(job) + } + fn initialize_parameter_buffer(&mut self) -> Result { + let region = |owner: usize| GpuRegion::new(self.objects[owner].gpu_va()?, self.objects[owner].size()) + .map_err(|_| EINVAL); + let pages = region(PARAMETER_PAGE_LIST)?; + let blocks = region(PARAMETER_BLOCK_LIST)?; + let control = queue::FirmwareVa::new(self.objects[BLOCK_CONTROL].va()).map_err(|_| EINVAL)?; + let counter = queue::FirmwareVa::new(self.objects[s::BM_COUNTER].va()).map_err(|_| EINVAL)?; + let manager = parameter::Manager::new(pages, blocks, control, counter, PB_BLOCKS).map_err(|_| EINVAL)?; + let mut bytes = [0; parameter::MANAGER_SIZE]; + manager.encode(&mut bytes).map_err(|_| EINVAL)?; + for group in 0..PB_GROUPS { + let buffer = &self.objects[PB_FIRST + group]; + let backing = parameter::Backing::new(GpuRegion::new(buffer.gpu_va()?, buffer.size()).map_err(|_| EINVAL)?) + .map_err(|_| EINVAL)?; + for block in 0..PB_BLOCKS_PER_GROUP { + self.objects[PARAMETER_BLOCK_LIST].write((group * PB_BLOCKS_PER_GROUP + block) * 8, + &backing.block(block).map_err(|_| EINVAL)?)?; + } + for page in 0..PB_BLOCKS_PER_GROUP * parameter::PAGES_PER_BLOCK { + self.objects[PARAMETER_PAGE_LIST].u32((group * PB_BLOCKS_PER_GROUP * parameter::PAGES_PER_BLOCK + page) * 4, + backing.page(page).map_err(|_| EINVAL)?)?; + } + } + self.objects[BUFFER_MANAGER].write(0, &bytes) + } + fn initialize_state(&mut self) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); + let notification = state::Notifier { threshold: address(s::EVENT_COUNT)?, context: self._binding.slot() }; + let list_owner = address(JOB_LIST)?; + let mut context = [0; state::CONTEXT_SIZE]; + state::context(&mut context).map_err(|_| EINVAL)?; + self.objects[GPU_CONTEXT].write(0, &context)?; + let mut notifier = [0; state::NOTIFIER_SIZE]; + notification.encode(&mut notifier).map_err(|_| EINVAL)?; + self.objects[NOTIFIER].write(0, ¬ifier)?; + let mut list = [0; state::JOB_LIST_SIZE]; + state::job_list(&mut list, list_owner).map_err(|_| EINVAL)?; + self.objects[JOB_LIST].write(0, &list)?; + let mut manager = [0; state::MANAGER_STATE_SIZE]; + state::block_control(&mut manager, PB_BLOCKS, PB_BLOCKS).map_err(|_| EINVAL)?; + self.objects[BLOCK_CONTROL].write(0, &manager)?; + state::manager_misc(&mut manager).map_err(|_| EINVAL)?; + self.objects[MANAGER_MISC].write(0, &manager)?; + for (owner, value) in [(TA_POOL_COUNTER, 1), (FRAGMENT_POOL_COUNTER, 1), + (s::EVENT_COUNT, 2), (s::BM_COUNTER, 1), + (s::TA_STAMP, timeline::stamp(s::STAMP_TA - 256, 0)), (s::TA_FW_STAMP, timeline::stamp(s::STAMP_TA - 256, 0)), + (s::FRAGMENT_STAMP, timeline::stamp(s::STAMP_FRAGMENT - 256, 0)), (s::FRAGMENT_FW_STAMP, timeline::stamp(s::STAMP_FRAGMENT - 256, 0))] { + let mut bytes = [0; state::MANAGER_STATE_SIZE]; + let bytes = bytes.get_mut(..self.objects[owner].size()).ok_or(EINVAL)?; + state::counter(bytes, value).map_err(|_| EINVAL)?; + self.objects[owner].write(0, bytes)?; + } + Ok(()) + } + fn initialize_queues(&mut self) -> Result { + for owner in &RENDER_QUEUES { + let address = |index: usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_| EINVAL); + let info = queue::QueueInfo { + state: address(owner.state)?, ring: address(owner.ring)?, + job_list: address(JOB_LIST)?, gpu_scratch: address(owner.scratch)?, + context: address(GPU_CONTEXT)?, uuid: owner.uuid, + }; + let seed_addresses=if owner.tiler { + [self.init_bm.va(),self.passes[0].get(P::TilerCommand).va()] + }else{[self.passes[0].get(P::TilerToFragment).va(),self.passes[0].get(P::FragmentCommand).va()]}; + let seeds=[queue::FirmwareVa::new(seed_addresses[0]).map_err(|_|EINVAL)?, + queue::FirmwareVa::new(seed_addresses[1]).map_err(|_|EINVAL)?]; + let mut bytes = [0; queue::INFO_SIZE]; + info.encode(&mut bytes).map_err(|_| EINVAL)?; + self.objects[owner.info].write(0, &bytes)?; + let mut state = [0; queue::STATE_SIZE]; + queue::RingState { capacity: queue::QUALIFIED_CAPACITY, done: 0, read: 0, write: 0 } + .encode(&mut state).map_err(|_| EINVAL)?; + self.objects[owner.state].write(0, &state)?; + for index in 0..queue::QUALIFIED_CAPACITY { + let entry = queue::ring_entry(index, queue::QUALIFIED_CAPACITY, &seeds).map_err(|_| EINVAL)?; + self.objects[owner.ring].write(index as usize * 8, &entry)?; + } + } + Ok(()) + } + fn write_barrier(&mut self, owner: P, command: sync::Barrier) -> Result { + let mut bytes = [0; sync::BARRIER_SIZE]; + command.encode(&mut bytes).map_err(|_| EINVAL)?; + self.passes[self.slot].get_mut(owner).write(0, &bytes) + } + // Construct synchronization from the actual owners after resetting a pass, + // before publishing its ring entries. InitBM is shared and dispatched only + // for draw one; rewriting its bytes after retirement preserves the old path. + fn prepare_sync(&mut self, early: bool, draw: u64, + predecessor:Option) -> Result { + let address = |owner: usize| queue::FirmwareVa::new(self.objects[owner].va()).map_err(|_| EINVAL); + let manager = address(BUFFER_MANAGER)?; + let ta_stamp = address(s::TA_FW_STAMP)?; + let fs_stamp = address(s::FRAGMENT_FW_STAMP)?; + let mut init = [0; sync::INIT_BM_SIZE]; + sync::InitBufferManager { context: self._binding.slot(), slot: 0, + block_count: PB_BLOCKS, manager, stamp: timeline::stamp(s::STAMP_TA - 256, 1) } + .encode(&mut init).map_err(|_| EINVAL)?; + self.init_bm.write(0, &init)?; + let ta_value = timeline::stamp(s::STAMP_TA - 256, draw); + let fs_value = timeline::stamp(s::STAMP_FRAGMENT - 256, draw); + let previous=predecessor.unwrap_or(sync::CompletionDependency { + stamp:if early {ta_stamp} else {fs_stamp}, + value:timeline::stamp((if early {s::STAMP_TA} else {s::STAMP_FRAGMENT})-256,draw-1), + event:u32::from(!early), + }); + self.write_barrier(P::TilerToFragment, sync::Barrier { + stamp: ta_stamp, wait_value: ta_value, event: 0, self_value: fs_value, + uuid: 0x3d0000, dependency: sync::Dependency::TilerToFragment })?; + self.write_barrier(P::TilerDependency, sync::Barrier { + stamp: previous.stamp, wait_value: previous.value, + event: previous.event, self_value: ta_value, + uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass })?; + self.write_barrier(P::FragmentDependency, sync::Barrier { + stamp: fs_stamp, wait_value: timeline::stamp(s::STAMP_FRAGMENT - 256, draw - 1), event: 1, self_value: fs_value, + uuid: DEPENDENCY_UUID, dependency: sync::Dependency::PreviousPass }) + } + // Prepare each private command after its retired slot is reset. The + // reusable host staging allocation is copied before the next slot is built. + fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { + use crate::m3_compute_layout::GpuVa; + let pass=&self.passes[self.slot]; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let gpu=|field:P| GpuVa::new(pass.get(field).gpu_va()?).map_err(|_|EINVAL); + let seq=pass.get(P::TilerSequence); + let command=pass.get(P::TilerCommand); + let g=Self::geometry(r,self.clusters)?; + if g.tpc_bytes>pass.get(P::Tpc).size() as u64 + || g.tilemap_bytes>pass.get(P::Tilemap).size() as u64 {return Err(E2BIG);} + // The old per-slot register keeps its seed entry, independent of the + // draw-dependent RenderScene reference. Do not change this policy here. + let initial_scene_entry=queue::FirmwareVa::new(self.objects[s::BM_SCENES].va() + +(self.slot as u64+1)*4).map_err(|_|EINVAL)?; + let value=tiler::Command { + context:self._binding.slot(),register_set:self.registers, + notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?, + scene:fw(P::Scene)?,empty:shared(547)?,gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?, + sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?,pool:shared(UMA_POOLS[0].manager)?, + scratch:fw(P::TilerScratch)?,stamp:shared(s::TA_STAMP)?,fw_stamp:shared(s::TA_FW_STAMP)?, + stamp_value:timeline::stamp(s::STAMP_TA - 256, draw),timestamps:[fw(P::TilerStart)?,fw(P::TilerEnd)?], + user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?], + tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?, + scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES, + vdm:crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, + multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, + }; + value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::TilerCommand).write(0,&self.tiler_bytes) + } + fn prepare_sequences(&mut self,draw:u64)->Result { + for fs in [false,true] { + let pass=&self.passes[self.slot]; + let region=|field:P| {let b=pass.get(field);Region::new(b.va(),b.size()).map_err(|_|EINVAL)}; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let target=if fs{P::FragmentSequence}else{P::TilerSequence}; + let notifier=&self.objects[NOTIFIER]; + let owners=sequence::Owners { + command:region(if fs{P::FragmentCommand}else{P::TilerCommand})?,sequence:region(target)?, + stats:self.stats,notifier:Region::new(notifier.va(),notifier.size()).map_err(|_|EINVAL)?, + queue:shared(if fs{s::FRAGMENT_QUEUE}else{s::TA_QUEUE})?,scene:fw(P::Scene)?, + manager:shared(BUFFER_MANAGER)?,pool:shared(UMA_POOLS[usize::from(fs)].manager)?, + fw_stamp:shared(if fs{s::FRAGMENT_FW_STAMP}else{s::TA_FW_STAMP})?, + context:self._binding.slot(),stamp:timeline::stamp((if fs{s::STAMP_FRAGMENT}else{s::STAMP_TA}) - 256, draw), + }; + let timed=self.stage_timestamps[usize::from(fs)]; + let length=if fs { + let b=pass.get(P::FragmentScratch); + let scratch=GpuRegion::new(b.gpu_va()?,b.size()).map_err(|_|EINVAL)?; + sequence::Fragment{owners,scratch,queue_command_count:timeline::previous(draw)}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)? + }else{sequence::Tiler{owners,scratch:fw(P::TilerScratch)?}.encode_timestamps(&mut self.sequence_bytes,timed).map_err(|_|EINVAL)?}; + self.passes[self.slot].get_mut(target).write(0,&self.sequence_bytes)?; + self.passes[self.slot].get_mut(if fs{P::FragmentCommand}else{P::TilerCommand}).u32(sequence::command_length_offset(fs),length as u32)?; + } + Ok(()) + } + fn view_owner(&mut self,owner:ViewOwner)->&mut Buffer { + match owner {ViewOwner::Shared(index)=>&mut self.objects[index], + ViewOwner::Pass{slot,field}=>self.passes[slot].get_mut(field)} + } + fn patch_context(&mut self)->Result { + let context=self._binding.slot(); + self.objects[NOTIFIER].u32(state::NOTIFIER_CONTEXT, context)?; + Ok(()) + } + /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. + fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { + use crate::agx_render::{Geometry,Utile}; + let x=if r.utile_width==16 {Utile::Pixels16} else {Utile::Pixels32}; + let y=if r.utile_height==16 {Utile::Pixels16} else {Utile::Pixels32}; + let mut g=Geometry::new_layered(u32::from(r.width),u32::from(r.height),x,y,clusters,r.layers).map_err(|_|EINVAL)?; + g.set_samples(r.samples).map_err(|_|EINVAL)?; + Ok(g) + } + fn prepare_fragment(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { + use crate::{m3_compute_layout::GpuVa,agx_render_state::{Program,DepthStencil}}; + let pass=&self.passes[self.slot]; + let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); + let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); + let gpu=|field:P| GpuVa::new(pass.get(field).gpu_va()?).map_err(|_|EINVAL); + let seq=pass.get(P::FragmentSequence); + let command=pass.get(P::FragmentCommand); + let geometry=Self::geometry(r,self.clusters)?; + let mask=if r.flags&16!=0 {u64::MAX} else {u32::MAX as u64}; + let program=|p:agx_uapi::UapiProgram|->Result {Ok(Program { + address:usc.checked_add(u64::from(p.usc&!63)).ok_or(EOVERFLOW)?,resources:p.resource_spec&mask, + })}; + let tilebuffer_control=(u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)).div_ceil(2048) + |(u32::from(r.samples.trailing_zeros() as u8)<<17); + let tile_mode=0x280|u64::from(r.layers>1)|if r.flags&2!=0 {1<<16}else{0}; + let value=fragment::Command { + context:self._binding.slot(),register_set:self.registers, + sequence:Region::new(seq.va(),seq.size()).map_err(|_|EINVAL)?, + notifier:shared(NOTIFIER)?,manager:shared(BUFFER_MANAGER)?,scene:fw(P::Scene)?,empty:shared(547)?, + gpu_alias:GpuVa::new(command.gpu_va()?).map_err(|_|EINVAL)?,tilemap:gpu(P::Tilemap)?, + heap:gpu(P::HeapMetadata)?,auxiliary:gpu(P::Auxiliary)?,scene_user:gpu(P::SceneUser)?, + scratch:gpu(P::FragmentScratch)?,pool:shared(UMA_POOLS[1].manager)?,stamp:shared(s::FRAGMENT_STAMP)?, + fw_stamp:shared(s::FRAGMENT_FW_STAMP)?,stamp_value:timeline::stamp(s::STAMP_FRAGMENT - 256, draw), + timestamps:[fw(P::FragmentStart)?,fw(P::FragmentEnd)?],user_timestamps:[fw(P::FragmentUserStart)?,fw(P::FragmentUserEnd)?], + state:fragment::State { + geometry,multisample:r.multisample_control,merge_upper:[r.merge_upper_x,r.merge_upper_y], + eot:program(r.end_of_tile)?,background:program(r.background)?, + partial_eot:program(r.partial_end_of_tile)?,partial_background:program(r.partial_background)?, + scissor:r.scissor_base,depth_bias:r.depth_bias_base,query:r.occlusion_query_base, + isp_control:0xc000|(r.flags&(1<<18)),tilebuffer_control,mirror_tilebuffer_control:tilebuffer_control, + tile_mode,mirror_tile_mode:tile_mode as u32, + depth:DepthStencil{base:r.depth.base,stride:u64::from(r.depth.stride)}, + stencil:DepthStencil{base:r.stencil.base,stride:u64::from(r.stencil.stride)}, + depth_compression:DepthStencil{base:r.depth.compression_base,stride:u64::from(r.depth.compression_stride)}, + stencil_compression:DepthStencil{base:r.stencil.compression_base,stride:u64::from(r.stencil.compression_stride)}, + zls_control:r.zls_control,depth_dimensions:r.depth_dimensions,depth_clear:r.depth_clear, + stencil_clear:r.stencil_clear,sample_size:u32::from(r.sample_size),process_empty_tiles:r.flags&2!=0, + }, + }; + value.encode(&mut self.fragment_bytes).map_err(|_|EINVAL)?; + self.passes[self.slot].get_mut(P::FragmentCommand).write(0,&self.fragment_bytes) + } + pub(crate) fn set_user_timestamps(&mut self,addresses:[[u64;2];2])->Result { + if addresses.iter().enumerate().any(|(stage,pair)|*pair!=[0;2] && !self.stage_timestamps[stage]) {return Err(EINVAL);} + for (stage,(command,sequence,offset,start,end)) in [ + (P::TilerCommand,P::TilerSequence,sequence::ta::USER_PAIR,sequence::ta::TIMESTAMP_START,sequence::ta::TIMESTAMP_END), + (P::FragmentCommand,P::FragmentSequence,sequence::fragment::USER_PAIR,sequence::fragment::TIMESTAMP_START,sequence::fragment::TIMESTAMP_END) + ].into_iter().enumerate() { + let pair=addresses[stage]; + if !self.stage_timestamps[stage] {continue;} + self.passes[self.slot].get_mut(command).u64(offset,pair[0])?; + self.passes[self.slot].get_mut(command).u64(offset+8,pair[1])?; + let pointer=if pair==[0,0] {0} else {self.passes[self.slot].get_mut(command).va()+offset as u64}; + self.passes[self.slot].get_mut(sequence).u64(start+sequence::USER_TIMESTAMP_POINTER,pointer)?; + self.passes[self.slot].get_mut(sequence).u64(end+sequence::USER_TIMESTAMP_POINTER,pointer)?; + } + crate::agx_memory::publish();Ok(()) + } + fn at(base:usize,index:usize)->usize {(base+index)%pass_layout::SLOTS} + /// GPU time of the batch of `count` passes starting at pass slot `base`. + pub(crate) fn gpu_ns(&mut self,base:usize,count:usize)->Result { + let start=self.passes[base].get_mut(P::TilerStart).read_u64(0)?; + let end=self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?; + Ok(if start!=0 && end>=start {(end-start)*1000/24} else {0}) + } + /// The batch's first tiler start and last fragment end, in 24 MHz GPU timestamp ticks. + pub(crate) fn gpu_ticks(&mut self,base:usize,count:usize)->Result<(u64,u64)> { + Ok((self.passes[base].get_mut(P::TilerStart).read_u64(0)?, + self.passes[Self::at(base,count-1)].get_mut(P::FragmentEnd).read_u64(0)?)) + } + /// Read retained timestamps only after exact batch retirement. The first + /// TA and final fragment delimit the batch in the shared 24 MHz counter. + pub(crate) fn batch_gpu_span(&mut self)->Result<[u64;2]> { + if self.batch_count == 0 {return Err(EINVAL);} + Ok([self.passes[self.base].get_mut(P::TilerStart).read_u64(0)?, + self.passes[Self::at(self.base,self.batch_count-1)].get_mut(P::FragmentEnd).read_u64(0)?]) + } + pub(crate) fn stage_ns(&mut self,base:usize,count:usize)->Result<[u64;3]> { + let mut total=[0;3]; + for index in 0..count { + let stages=self.slot_stage_ns(base,count,index)?; + for i in 0..3 {total[i]+=stages[i];} + } + Ok(total) + } + pub(crate) fn slot_stage_ns(&mut self,base:usize,count:usize,index:usize)->Result<[u64;3]> { + if index>=count {return Err(EINVAL);} + let slot=Self::at(base,index); + let duration=|a:u64,b:u64| if a!=0 && b>=a {(b-a)*1000/24} else {0}; + let fs_start=self.passes[slot].get_mut(P::FragmentStart).read_u64(0)?; + let fs_end=self.passes[slot].get_mut(P::FragmentEnd).read_u64(0)?; + let ta_start=self.passes[slot].get_mut(P::TilerStart).read_u64(0)?; + let ta_end=self.passes[slot].get_mut(P::TilerEnd).read_u64(0)?; + Ok([duration(ta_start,ta_end),duration(fs_start,fs_end),duration(ta_end,fs_start)]) + } + pub(crate) fn early_count(&self)->u32 {self.early_count} + pub(crate) fn heads(&self)->[u16;2] {self.heads} + pub(crate) fn ordinal(&self)->u64 {self.draw} + /// The current batch: first pass slot, pass count and last draw number. + pub(crate) fn batch(&self)->(usize,usize,u64) {(self.base,self.batch_count,self.draw)} + /// Whether draws 1..=`draw` have completed both stages (TA and fragment stamps, host and + /// firmware copies, have reached `draw`'s values). Use signed modular ordering + /// across stamp wrap; the runtime bounds outstanding passes to SLOTS. + pub(crate) fn complete_through(&mut self,draw:u64)->Result { + if draw==0 {return Ok(true);} + for (stamp,fw,value) in [(s::TA_STAMP,s::TA_FW_STAMP,timeline::stamp(s::STAMP_TA-256,draw)), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,timeline::stamp(s::STAMP_FRAGMENT-256,draw))] { + if !timeline::reached(self.objects[stamp].read_u32(0)?,value) || !timeline::reached(self.objects[fw].read_u32(0)?,value) {return Ok(false);} + } + Ok(true) + } + /// Whether any draw has been published (the first one initializes the buffer manager). + pub(crate) fn started(&self)->bool {self.draw>0} + pub(crate) fn first(&self)->bool {self.draw==self.batch_count as u64} + /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then + /// the VM must already be bound, no storage may grow and the slots must not be in flight + /// (the caller keeps the in-flight passes plus this batch within SLOTS). + pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + commands:&[crate::m3_submit::Command],base:usize,overlap:bool)->Result { + if base>=pass_layout::SLOTS || (!overlap && !self.complete()?) {return Err(EBUSY);} + if overlap { + if !self._binding.matches(vm) {return Err(EBUSY);} + for (index,&command) in commands.iter().enumerate() { + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let g=Self::geometry(r,self.clusters)?; + let slot=Self::at(base,index); + if g.tpc_bytes>self.passes[slot].get(P::Tpc).size() as u64 + || g.tilemap_bytes>self.passes[slot].get(P::Tilemap).size() as u64 {return Err(EBUSY);} + } + } + if commands.is_empty() || commands.len()>pass_layout::SLOTS {return Err(E2BIG);} + self.draw.checked_add(commands.len() as u64).ok_or(EOVERFLOW)?; + // Validate every pass before changing any queue or completion state. + for &command in commands { + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + Self::geometry(r,self.clusters)?; + crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; + } + if !self._binding.matches(vm) { + let next=if let Some(index)=self.cached_views.iter().position(|v|v.binding.matches(vm)) { + self.cached_views.remove(index).map_err(|_|EIO)? + } else { + let binding=uat.bind(vm)?; + let mut views=KVec::new();let mut aliases=KVec::new(); + for i in 0..s::COUNT { + if s::allocation(i).map_err(|_|EINVAL)?.space==Space::ClientGpu { + views.push((ViewOwner::Shared(i),self.objects[i].map_gpu_view(vm)?),GFP_KERNEL)?; + } + } + for (slot,pass) in self.passes.iter_mut().enumerate() { + for field in pass_layout::FIELDS { + let a=pass_layout::board_allocation(slot,field,self.clusters).map_err(|_|EINVAL)?; + if a.space==Space::ClientGpu {views.push((ViewOwner::Pass{slot,field},pass.get_mut(field).map_gpu_view(vm)?),GFP_KERNEL)?;} + if a.gpu_alias.is_some() {aliases.push(pass.get_mut(field).map_gpu_view(vm)?,GFP_KERNEL)?;} + } + } + CachedViews{binding,aliases,views} + }; + // Finish fallible allocations before replacing the current owner. + let old_views=KVec::with_capacity(next.views.len(),GFP_KERNEL)?; + self.cached_views.reserve(1,GFP_KERNEL)?; + if self.cached_views.len()==3 {drop(self.cached_views.remove(0).map_err(|_|EIO)?);} + let mut old=CachedViews { + binding:core::mem::replace(&mut self._binding,next.binding), + aliases:core::mem::replace(&mut self._aliases,next.aliases), + views:old_views, + }; + for (i,view) in next.views { + old.views.push((i,self.view_owner(i).swap_gpu_view(view)?),GFP_KERNEL)?; + } + self.cached_views.push(old,GFP_KERNEL)?; + } + // Growth is legal only after exact retirement, and before append. + // Cached VM mappings must be dropped before freeing the old backing. + // Partial allocation failure leaves retired queue/stamp state intact; + // already-grown buffers are reusable on the next submission. + for (index,&command) in commands.iter().enumerate() { + let slot=Self::at(base,index); + let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let g=Self::geometry(r,self.clusters)?; + for (field,base,bytes) in [(P::Tpc,s::TPC_LABEL,g.tpc_bytes),(P::Tilemap,s::TILEMAP_LABEL,g.tilemap_bytes)] { + let size=usize::try_from(bytes).map_err(|_|EOVERFLOW)?; + if size>self.passes[slot].get(field).size() { + let buffer=Buffer::tiler(dev,uat.kernel_vm(),vm,size)?; + self.cached_views.clear(); + dev_info!(dev.as_ref(),"M3 render arena growth: slot={} object={} {} -> {} bytes GPU={:#x}\n", + slot,base,self.passes[slot].get(field).size(),size,buffer.gpu_va()?); + *self.passes[slot].get_mut(field)=buffer; + } + } + } + self.checkpoint=(self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base); + self.batch_count=0; + self.base=base; + Ok(()) + } + /// Restore producer state only when no batch is in flight. Running firmware can + /// consume append's WRITE updates without a new doorbell, so overlap cannot abort. + pub(crate) fn abort_batch(&mut self)->Result { + (self.draw,self.heads,self.batch_count,self.slot,self.early_count,self.base)=self.checkpoint; + self.objects[s::BM_COUNTER].u32(0,self.draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(self.draw,2))?; + for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { + self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; + } + crate::agx_memory::publish(); + Ok(()) + } + /// Append only into caller-reserved free pass slots. With overlap, WRITE updates + /// are immediately visible to running firmware, even before a new doorbell. + /// Each append selects disjoint host-mutated pass storage. A TA dependency + /// retains full fragment-to-next-TA ordering for arbitrary resource hazards. + pub(crate) fn append(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,coalesce_stamps:bool,timestamps:[bool;2])->Result { + self.append_after(r,usc,coalesce_stamps,None,timestamps) + } + pub(crate) fn completion_dependency(&self)->Result { + Ok(sync::CompletionDependency { + stamp:queue::FirmwareVa::new(self.objects[s::FRAGMENT_FW_STAMP].va()).map_err(|_|EINVAL)?, + value:timeline::stamp(s::STAMP_FRAGMENT-256,self.draw),event:1, + }) + } + pub(crate) fn append_after(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64, + coalesce_stamps:bool,predecessor:Option,timestamps:[bool;2])->Result { + if predecessor.is_some_and(|p|p.event!=2) {return Err(EINVAL);} + if self.batch_count>=pass_layout::SLOTS { + pr_err!("M3 render batch limit: batch={} slots={}\n", + self.batch_count,pass_layout::SLOTS); + return Err(E2BIG); + } + self.slot=Self::at(self.base,self.batch_count); + self.stage_timestamps=timestamps; + // Retain the conservative stage-scope permission. Only a pass after the + // first of its batch may overlap its predecessor's stages. Explicit + // start timestamps keep the original full-order execution boundary. + let early=predecessor.is_none() && self.batch_count>0 && *crate::module_parameters::m3_early_tiling.value()!=0 + && r.flags&(1<<5)!=0 && r.vertex_timestamps.start.handle==0 + && r.fragment_timestamps.start.handle==0; + if early {self.early_count+=1;} + // Only reset per-draw objects. Pool allocators, PB lists, queues, + // notification state and previous stamps retain firmware ownership. + self.passes[self.slot].reset(&Self::geometry(r,self.clusters)?)?; + self.init_bm.fill(0)?; // retain the prior shared InitBM reset before encoding + self.draw=self.draw.checked_add(1).ok_or(EOVERFLOW)?; + self.batch_count+=1; + let draw=self.draw; + self.objects[s::BM_COUNTER].u32(0,draw as u32)?; + self.objects[s::EVENT_COUNT].u32(0,timeline::events(draw,2))?; + let entry_offset = (self.draw % u64::from(MANAGER_SCENE_COUNT)) as usize * 4; + let scene_list = &mut self.objects[s::BM_SCENES]; + if entry_offset + 4 > scene_list.size() { return Err(ERANGE); } + let manager_entry = queue::FirmwareVa::new(scene_list.va().checked_add(entry_offset as u64) + .ok_or(EOVERFLOW)?).map_err(|_| EINVAL)?; + let manager_misc = queue::FirmwareVa::new(self.objects[MANAGER_MISC].va()).map_err(|_| EINVAL)?; + let user_buffer = self.passes[self.slot].get_mut(P::SceneUser); + let user_buffer = scene::UserBuffer::new(user_buffer.gpu_va()?, user_buffer.size()).map_err(|_| EINVAL)?; + let mut bytes = [0; scene::SIZE]; + scene::Scene { manager_entry, manager_misc, user_buffer }.encode(&mut bytes).map_err(|_| EINVAL)?; + self.passes[self.slot].get_mut(P::Scene).write(0, &bytes)?; + self.prepare_tiler(r,draw)?; + self.patch_context()?; + self.prepare_fragment(r,usc,draw)?; + self.prepare_sequences(draw)?; + self.prepare_sync(early, draw, predecessor)?; + // Encode before either ring WRITE exposes the command to firmware. + // The caller retains every pass owner until the final flushed stamps, + // queues, events and engine-idle checks retire this entire batch. + if coalesce_stamps { + self.passes[self.slot].get_mut(P::TilerCommand).u32(tiler::FLUSH_STAMPS,0)?; + self.passes[self.slot].get_mut(P::FragmentCommand).u32(fragment::FLUSH_STAMPS,0)?; + } + let dep=P::TilerDependency; + let fs_dep=P::FragmentDependency; + let ta=self.passes[self.slot].get_mut(P::TilerCommand).va(); + let fragment=self.passes[self.slot].get_mut(P::FragmentCommand).va(); + let barrier=self.passes[self.slot].get_mut(P::TilerToFragment).va(); + let dependency=self.passes[self.slot].get(dep).va(); + let init=self.init_bm.va(); + let ta_commands=[init,dependency,ta]; + let fs_commands=[self.passes[self.slot].get(fs_dep).va(),barrier,fragment]; + for (stage,ring,pointers,addresses) in [ + (0,s::TA_RING,s::TA_POINTERS,&ta_commands[usize::from(draw!=1)..]), + (1,s::FRAGMENT_RING,s::FRAGMENT_POINTERS,&fs_commands[usize::from(!early)..])] { + for &address in addresses { + self.objects[ring].u64(usize::from(self.heads[stage])*8,address)?; + self.heads[stage]=(self.heads[stage]+1)%0x500; + } + crate::agx_memory::publish(); + self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; + } + crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) + } + /// Emitted only after both events, stamp/queue checks, clear faults, + /// idle engines and consumed firmware pipes have been checked by Runtime. + /// The existing TA/fragment descriptors request completion stamp flushes. + pub(crate) fn progress(&mut self,dev:&driver::AsahiDevice)->Result { + // Keep summaries sparse at high throughput, but emit at least once a + // second while completed batches advance. Batched draw counts can skip + // multiples of 128; a count-only gate can starve the watchdog observer. + if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + let now = ::ktime_get(); + if self.last_progress_ns != 0 && now - self.last_progress_ns < 1_000_000_000 + && self.draw != 1 && self.draw % 128 != 0 + && !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + return Ok(()); + } + self.last_progress_ns = now; + dev_info!(dev.as_ref(),"M3_RETIRED draw={} ta={:x}/{:x} fragment={:x}/{:x} queues={}/{} read={}/{} idle=1 cache_flush=1 events=2 faults=0 ordered_barrier=1 batch={} early={}\n", + self.draw,self.objects[s::TA_STAMP].read_u32(0)?,self.objects[s::TA_FW_STAMP].read_u32(0)?, + self.objects[s::FRAGMENT_STAMP].read_u32(0)?,self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?, + self.objects[s::TA_POINTERS].read_u32(queue::DONE)?,self.objects[s::FRAGMENT_POINTERS].read_u32(queue::DONE)?, + self.objects[s::TA_POINTERS].read_u32(queue::READ)?,self.objects[s::FRAGMENT_POINTERS].read_u32(queue::READ)?,self.batch_count,self.early_count); + Ok(()) + } + pub(crate) fn queues(&self)->[u64;2] {[self.objects[s::TA_QUEUE].va(),self.objects[s::FRAGMENT_QUEUE].va()]} + /// Read retained command bytes only after exact batch retirement. + pub(crate) fn stamp_flush_masks(&mut self,base:usize,count:usize)->Result<[u16;2]> { + if base>=pass_layout::SLOTS || count==0 || count>pass_layout::SLOTS {return Err(EINVAL);} + let mut masks=[0;2]; + for slot in 0..count { + let pass=&mut self.passes[Self::at(base,slot)]; + for (stage,field,offset) in [(0,P::TilerCommand,tiler::FLUSH_STAMPS), + (1,P::FragmentCommand,fragment::FLUSH_STAMPS)] { + if pass.get_mut(field).read_u32(offset)?!=0 {masks[stage]|=1u16<Result<[bool;2]> { + let value=timeline::stamp(s::STAMP_FRAGMENT - 256,self.draw); + Ok([self.objects[s::FRAGMENT_STAMP].read_u32(0)?==value, + self.objects[s::FRAGMENT_FW_STAMP].read_u32(0)?==value]) + } + /// Observe each completion condition across both render queues independently. + pub(crate) fn completion_parts(&mut self)->Result<[bool;4]> { + let mut done=[true;4]; + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + done[0]&=self.objects[stamp].read_u32(0)?==value; + done[1]&=self.objects[fw].read_u32(0)?==value; + done[2]&=self.objects[pointers].read_u32(queue::DONE)?==u32::from(head); + done[3]&=self.objects[pointers].read_u32(queue::READ)?==u32::from(head); + } + Ok(done) + } + pub(crate) fn complete(&mut self)->Result { + for (stamp,fw,pointers,value,head) in [(s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS,timeline::stamp(s::STAMP_TA - 256, self.draw),self.heads[0]), + (s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS,timeline::stamp(s::STAMP_FRAGMENT - 256, self.draw),self.heads[1])] { + if self.objects[stamp].read_u32(0)?!=value || self.objects[fw].read_u32(0)?!=value + || self.objects[pointers].read_u32(queue::DONE)?!=u32::from(head) + || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} + } Ok(true) + } + pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + // Retain allocator state as well as payloads: an exhausted parameter + // pool can stall TA without producing an MMU fault. + for (index,bytes) in [(BLOCK_CONTROL,64),(s::BM_COUNTER,64),(MANAGER_MISC,64),(BUFFER_MANAGER,172)] { + for offset in (0..bytes).step_by(4) { + let value=self.objects[index].read_u32(offset)?; + dev_info!(dev.as_ref(),"M3_PB object={} offset={:#x} value={:#x}\n",index,offset,value); + } + } + dev_info!(dev.as_ref(),"M3 fault render VM={} slot={} root={:#x} draw={} batch={}\n",self._binding.vm_id(),self._binding.slot(),self._binding.root(),self.draw,self.batch_count); + for index in 0..self.batch_count { + let slot=Self::at(self.base,index); + let command=self.passes[slot].get_mut(P::TilerCommand); + for index in 0..tiler::REGISTER_COUNT { + let offset=tiler::REGISTERS+index*tiler::REGISTER_STRIDE; + if command.read_u32(offset)?==0x1c880 { + dev_info!(dev.as_ref(),"M3 fault pass slot={} compact_vdm={:#x}\n",slot,command.read_u64(offset+4)?); + } + } + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + // Preserve historic diagnostic IDs while selecting named owners. + for (index,bytes) in s::fault_buffers() { + let b=match index {s::TILEMAP_LABEL=>self.passes[0].get_mut(P::Tilemap), + s::HEAP_METADATA_LABEL=>self.passes[0].get_mut(P::HeapMetadata),_=>&mut self.objects[index]}; + for offset in (0..bytes).step_by(16) { + let lo=b.read_u64(offset)?; + let hi=b.read_u64(offset+8)?; + if lo!=0 || hi!=0 {dev_info!(dev.as_ref(),"M3_BUFFER object={} offset={:#x} {:016x} {:016x}\n",index,offset,lo,hi);} + } + } + } + for (name,stamp,fw,pointers) in [("TA",s::TA_STAMP,s::TA_FW_STAMP,s::TA_POINTERS), + ("fragment",s::FRAGMENT_STAMP,s::FRAGMENT_FW_STAMP,s::FRAGMENT_POINTERS)] { + dev_info!(dev.as_ref(),"M3 {} stamps={:#x}/{:#x} done={} read={}\n",name, + self.objects[stamp].read_u32(0)?,self.objects[fw].read_u32(0)?, + self.objects[pointers].read_u32(queue::DONE)?,self.objects[pointers].read_u32(queue::READ)?); + } Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_resources.rs b/drivers/gpu/drm/asahi/m3/m3_resources.rs new file mode 100644 index 00000000000000..bfc94fdfa0ca8d --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_resources.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! M3 GPU resources admitted against the measured boot contract (J514S on T6030). +pub(crate) use crate::agx_resources::{Region, Resources}; + +use kernel::{bindings, c_str, io::resource::Resource, of, prelude::*}; + +/// Live overlays are not inserted in OF's boot-time reserved_mem registry. +/// Keep the ordinary lookup for every static handoff. A no-map node is +/// accepted only when its range is outside all Linux System RAM. +pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result { + if let Ok(r) = node.reserved_mem_region_to_resource_byname(name) { + return Ok(r); + } + if let Some(found) = crate::m3_board::static_region(node, name) { + return found.map(|(r, _)| r); + } + // Only for a GPU node whose compatible is exactly that of an M3 SoC table. + let compatible: KVec = node.get_property(c_str!("compatible"))?; + let compatible = compatible.strip_suffix(b"\0").ok_or(EINVAL)?; + if !crate::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { + return Err(EINVAL); + } + let names: KVec = node.get_property(c_str!("memory-region-names"))?; + let index = names.split(|b| *b == 0).position(|n| n == name.to_bytes()).ok_or(EINVAL)?; + let region = node.parse_phandle(c_str!("memory-region"), index).ok_or(EINVAL)?; + let nomap: KVec = region.get_property(c_str!("no-map"))?; + let reg: KVec = region.get_property(c_str!("reg"))?; + if !nomap.is_empty() || reg.len()!=2 { return Err(EINVAL); } + let (base,size)=(reg[0],reg[1]); + if base < 1<<40 || size == 0 || (base|size)&0x3fff != 0 + || base.checked_add(size).ok_or(EOVERFLOW)? > 1<<42 { return Err(EINVAL); } + // SAFETY: read-only resource-tree query with checked numeric bounds. + if unsafe { bindings::region_intersects(base, size.try_into()?, bindings::IORESOURCE_SYSTEM_RAM as _, 0) } != bindings::REGION_DISJOINT as i32 { + return Err(EBUSY); + } + let raw = bindings::resource { + start: base, end: base+size-1, flags: bindings::IORESOURCE_MEM as _, + ..Default::default() + }; + // SAFETY: Resource is repr(transparent) over Opaque, which + // preserves layout. The initialized descriptor is owned, with no pointers + // into temporary data; its range is checked above. + Ok(unsafe { core::mem::transmute::(raw) }) +} + +pub(crate) fn from_device( + pdev: &kernel::platform::Device, + soc: &crate::m3_soc::Soc, +) -> kernel::error::Result { + crate::m3_board::admit(pdev, soc) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs new file mode 100644 index 00000000000000..3e8cbbfa937817 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! RTKit callbacks for the identified J514S firmware. The firmware advertises +//! its crash storage by physical address inside the reserved data segment. + +use crate::agx_resources::Region; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use kernel::{ + c_str, impl_has_hr_timer, + device::Core, + sync::Completion, + io::mem::{Mem, MemFlag}, + iosys_map::IoSysMapRef, + platform, + prelude::*, + soc::apple::rtkit, + sync::{aref::ARef, Arc, ArcBorrow}, + time::{Delta, Monotonic, hrtimer::{HrTimer, HrTimerCallback, + HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode}}, +}; + +/// IRQ-safe notification tokens for the single GPU completion worker. +#[pin_data] +pub(crate) struct EventWait { + count: AtomicU64, + wait_us: AtomicU64, + overlap_cpu: AtomicBool, + cpu_preparations: AtomicU64, + sleeps: AtomicU64, + timer_wakes: AtomicU64, + #[pin] + arrived: Completion, + #[pin] + timer: HrTimer, +} + +impl EventWait { + fn new() -> Result> { + Arc::pin_init( + pin_init!(EventWait { + count: AtomicU64::new(0), + wait_us: AtomicU64::new(100), + overlap_cpu: AtomicBool::new(crate::m3_params::unlocked_wait()), + cpu_preparations: AtomicU64::new(0), + sleeps: AtomicU64::new(0), + timer_wakes: AtomicU64::new(0), + arrived <- Completion::new(), + timer <- HrTimer::new(), + }), + GFP_KERNEL, + ) + } + + fn record(&self) { + self.count.fetch_add(1, Ordering::Release); + self.arrived.complete(); + } + + /// Diagnostic tuning changes only the bounded resnapshot cadence, never + /// stamps, pipe retirement, event counts, fences or the two-second timeout. + /// Atomic publication permits root to compare policies in the same boot. + pub(crate) fn set_wait_us(&self, us: u64) -> Result { + if !(25..=1000).contains(&us) { return Err(EINVAL); } + self.wait_us.store(us, Ordering::Relaxed); + Ok(()) + } + + pub(crate) fn cpu_overlap(&self) -> bool { + self.overlap_cpu.load(Ordering::Relaxed) + } + pub(crate) fn set_cpu_overlap(&self, enabled: bool) { + self.overlap_cpu.store(enabled, Ordering::Relaxed); + } + + pub(crate) fn record_cpu_preparation(&self) { + self.cpu_preparations.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn snapshot(&self) -> (u64, u64, u64, u64, u64) { + (self.wait_us.load(Ordering::Relaxed), self.count.load(Ordering::Acquire), + self.sleeps.load(Ordering::Relaxed), self.timer_wakes.load(Ordering::Relaxed), + self.cpu_preparations.load(Ordering::Relaxed)) + } + + /// Sleep until a notification later than `seen` has been recorded or + /// the selected 25..1000 us elapse (default 100 us). A one-jiffy fallback alone takes up to 4 ms at HZ=250, + /// even when the GPU stamp arrives just after its notification. The timer + /// adds a bounded resnapshot without replacing IRQ wakeups or busy-waiting. + /// Only the single completion worker may call this method. + pub(crate) fn wait_past(self: &Arc, seen: u64) -> bool { + // Events can arrive while the worker is polling. Discard stale wakeup + // tokens before checking the sequence. A concurrent event either moves + // the sequence or leaves a token, including between the check and wait. + // Bound draining so a busy interrupt source cannot hold this worker. + for _ in 0..64 { + if !self.arrived.try_wait_for_completion() { + if self.count.load(Ordering::Acquire) != seen { return true; } + // The handle keeps EventWait alive and cancels synchronously + // on every return, before a subsequent wait can arm the timer. + self.sleeps.fetch_add(1, Ordering::Relaxed); + let timer = self.clone().start(Delta::from_micros( + self.wait_us.load(Ordering::Relaxed) as i64)); + // Retain one tick as a backup bound; normally either the real + // interrupt or our high-resolution timer supplies the token. + self.arrived.wait_for_completion_timeout(1); + drop(timer); + return self.count.load(Ordering::Acquire) != seen; + } + } + // Resnapshot GPU progress before draining another batch of tokens. + true + } +} + +impl HrTimerCallback for EventWait { + type Pointer<'a> = Arc; + + fn run(this: ArcBorrow<'_, Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { + // Wake only: a timer expiry is not a firmware completion notification. + this.timer_wakes.fetch_add(1, Ordering::Relaxed); + this.arrived.complete(); + HrTimerRestart::NoRestart + } +} + +impl_has_hr_timer! { + impl HasHrTimer for EventWait { + mode: RelativeMode, field: self.timer + } +} + +pub(crate) struct State { + dev: ARef, + drm: crate::driver::AsahiDevRef, + data: Region, + pub(crate) health: Arc, + claimed: AtomicBool, + /// Firmware event notifications received on the application endpoint. + /// The firmware sends one after a command's completion processing. + pub(crate) event_messages: AtomicU64, + pub(crate) events: Arc, +} + +/// Shared status contains no device references, so DRM files may retain it +/// after runtime teardown without retaining MMIO or creating a DRM cycle. +pub(crate) struct Health { + crashed: AtomicBool, + mapped: AtomicBool, + failed: AtomicBool, + progress: crate::agx_host_progress::Progress, + pub(crate) timing: crate::agx_timing_stats::Stats, +} + +impl Health { + pub(crate) fn healthy(&self) -> bool { + self.mapped.load(Ordering::Acquire) + && !self.crashed.load(Ordering::Acquire) + && !self.failed() + } + + pub(crate) fn set_gpu_pending(&self, pending: bool) { self.progress.set_pending(pending); } + + pub(crate) fn activity_snapshot(&self) -> (u64, u64, bool) { + let (generation, epoch) = self.progress.activity_snapshot(); + (generation, epoch, self.healthy()) + } + + pub(crate) fn record_completion(&self) { self.progress.record_completion(); } + + pub(crate) fn progress_snapshot(&self) -> (u64, u64, u64, bool) { + let (generation, count, timestamp) = self.progress.snapshot(); + (generation, count, timestamp, self.healthy()) + } + + pub(crate) fn failed(&self) -> bool { self.failed.load(Ordering::Acquire) } + + /// Whether RTKit reported a firmware crash. + pub(crate) fn crashed(&self) -> bool { self.crashed.load(Ordering::Acquire) } + + pub(crate) fn mark_failed(&self) { self.failed.store(true, Ordering::Release); } +} + +impl State { + pub(crate) fn new(dev: &platform::Device, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { + Ok(Arc::new( + Self { + dev: dev.into(), + drm, + data, + health: Arc::new(Health { + crashed: AtomicBool::new(false), + mapped: AtomicBool::new(false), + failed: AtomicBool::new(false), + progress: crate::agx_host_progress::Progress::new(), + timing: crate::agx_timing_stats::Stats::new(), + }, GFP_KERNEL)?, + claimed: AtomicBool::new(false), + event_messages: AtomicU64::new(0), + events: EventWait::new()?, + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn healthy(&self) -> bool { + self.health.healthy() + } +} + +pub(crate) struct FirmwareBuffer { + state: Arc, + mapping: Mem, + physical: usize, + offset: usize, + size: usize, +} + +impl Drop for FirmwareBuffer { + fn drop(&mut self) { + // Includes allocation failure in RTKit's wrapper after shmem_map + // returned: readiness must not survive loss of the actual mapping. + self.state.health.mapped.store(false, Ordering::Release); + } +} + +impl rtkit::Buffer for FirmwareBuffer { + fn iova(&self) -> Result { + Ok(self.physical) + } + fn buf(&mut self) -> Result> { + self.mapping.iosys_map(self.offset, self.size) + } +} + +pub(crate) struct Operations; + +#[kernel::macros::vtable] +impl rtkit::Operations for Operations { + type Data = Arc; + type Buffer = FirmwareBuffer; + + fn crashed(state: ArcBorrow<'_, State>, crashlog: Option<&[u8]>) { + state.health.crashed.store(true, Ordering::Release); + // Wake the serialized worker even if the firmware can no longer send + // its usual completion notification. Never take the runtime lock here. + state.events.record(); + crate::m3_completion::queue(&state.drm); + dev_err!( + state.dev.as_ref(), + "M3 G15S: firmware crashed, retained crashlog bytes={}\n", + crashlog.map_or(0, |b| b.len()) + ); + if let Some(bytes) = crashlog { + crate::t6031_start::capture_crashlog(state.dev.as_ref(), bytes); + } + } + + fn shmem_map( + state: ArcBorrow<'_, State>, + physical: usize, + size: usize, + ) -> Result { + // The only system buffer advertised by this admitted firmware is its + // preallocated crash log. Never reinterpret an arbitrary DVA as a PA. + let offset = physical + .checked_sub(state.data.base as usize) + .ok_or(EINVAL)?; + if size == 0 + || (physical | size) & 0xfff != 0 + || offset.checked_add(size).ok_or(EOVERFLOW)? > state.data.size as usize + { + return Err(EINVAL); + } + if state.claimed.swap(true, Ordering::AcqRel) { + return Err(EBUSY); + } + let node = state.dev.as_ref().of_node().ok_or(ENODEV)?; + let resource = crate::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; + if resource.start() != state.data.base || resource.size() != state.data.size { + return Err(EINVAL); + } + // SAFETY: The validated no-map data reservation belongs to this + // firmware session. This CPU mapping is retained by RTKit until ASC + // is stopped and callbacks drained; physical pages are never freed. + // An uncached CPU alias lets RTKit snapshot newly written crash data. + let mapping = unsafe { Mem::try_new(resource, MemFlag::WC.into()) }?; + dev_info!( + state.dev.as_ref(), + "M3 G15S: mapped firmware crash storage PA={:#x} size={:#x}\n", + physical, + size + ); + state.health.mapped.store(true, Ordering::Release); + Ok(FirmwareBuffer { + state: state.into(), + mapping, + physical, + offset, + size, + }) + } + + fn recv_message(state: ArcBorrow<'_, State>, endpoint: u8, message: u64) { + // Firmware retries its event notification until the shared rings are + // drained, including after the final userspace job. Never take the + // runtime mutex in the RTKit callback: enqueue the serialized worker. + if endpoint == 0x20 && message == 0x0042_0000_0000_0000 { + // Record the IRQ-safe token before publishing the external count. + // A concurrent waiter sees either the moved sequence or a token. + state.events.record(); + state.event_messages.fetch_add(1, Ordering::Release); + if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + dev_info!(state.dev.as_ref(), "M3 G15S_TIMING event monotonic_ns={} messages={}\n", + ::ktime_get(), + state.event_messages.load(Ordering::Acquire)); + } + crate::m3_completion::queue(&state.drm); + return; + } + dev_warn!( + state.dev.as_ref(), + "M3 G15S: RTKit application ep={:#x} message={:#018x}\n", + endpoint, + message + ); + } +} diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs new file mode 100644 index 00000000000000..08885f591c1406 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -0,0 +1,1246 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +use core::{mem::ManuallyDrop, sync::atomic::Ordering}; +use kernel::{device::Core, platform, prelude::*, soc::apple::rtkit, sync::Arc, + time::{Instant, Monotonic, Delta, delay::fsleep}}; +use crate::{driver, mmu, m3_device::Device, m3_config::Config, m3_rtkit}; + +#[path = "m3_batch_policy.rs"] +pub(crate) mod policy; +const _: () = assert!(policy::SLOTS == crate::m3_pass_layout::SLOTS); + +enum NativeJob {Compute(crate::m3_compute::Compute),Render(crate::m3_render::Render)} +/// Completion-worker outcome. +pub(crate) enum Service {Idle,Progress,Waiting(u64)} +/// The published batch: its commands in order, as (packet, command index). +struct Batch { + entries: KVec<(Arc,usize)>, + index: usize, kind: usize, first: crate::m3_submit::Command, + preparation_ns: i64, previous_events: u64, expected_events: u64, + start: Instant, measure: bool, doorbell_counter_ns: i64, + /// Render: first pass slot and the batch's last draw number. + base: usize, last_draw: u64, publication_tick:u64, + polling_ns: i64, polls: u64, stamp_ns: i64, first_event_ns: i64, events_ns: i64, + completion_ns: [i64;4], + /// First successful stamp observations and the preceding incomplete reads, in 24 MHz ticks. + fragment_host_tick:u64, fragment_fw_tick:u64, + fragment_host_pending_tick:u64, fragment_fw_pending_tick:u64, + pipes_idle_tick:u64, + coalesced_stamps: u64, + mixed: Option, +} +struct MixedBatch {compute_index:usize, renders:usize, computes:usize, dependencies:usize} +#[derive(Clone, Copy, Default)] +struct GeometryTiming { + width: u16, height: u16, samples: u8, count: u64, + prepare_ns: i64, active_ns: i64, gpu_ns: u64, + tiling_ns: u64, fragment_ns: u64, max_fragment_ns: u64, + polling_ns: i64, retirement_ns: i64, sleeping_ns: i64, polls: u64, + stamp_ns:i64, first_event_ns:i64, events_ns:i64, pickup_ns:i64, tail_ns:i64, + completion_ns: [i64;4], +} +impl NativeJob { + fn complete(&mut self)->Result {match self {Self::Compute(j)=>j.complete(),Self::Render(j)=>j.complete()}} + fn log(&mut self,dev:&driver::AsahiDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} + fn batch_gpu_span(&mut self)->Result<[u64;2]> {match self {Self::Compute(j)=>j.batch_gpu_span(),Self::Render(j)=>j.batch_gpu_span()}} +} +struct Inner { + transport: rtkit::RtKit, + state: Arc, + jobs: KVec, + packets: KVec>, + /// Packets not yet (fully) published: (packet, next command index), most urgent first. + pending: KVec<(Arc,usize)>, + /// Published batches, oldest first. More than one only with asahi.m3_pipeline_depth > 1. + active: KVec, + gpu_pending: bool, + active_power_last: Option>, + active_power_samples: u32, + fault_captured: bool, + timing: [[i64;7];2], + render_batches: [u64;crate::m3_pass_layout::SLOTS], + geometry: [GeometryTiming; 32], + geometry_overflow: u64, + /// Whether a T8122 job verdict line was already emitted for the current job, so the + /// execute-path fallback does not add a second one. T8122 only. + t8122_verdict: bool, + /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only. + boot_step: crate::t8122_start::BootStep, + /// Packets that joined a batch started by an earlier packet (debug summary). + coalesced_packets: u64, + coalesced_render_stamps: u64, + mixed_batches:u64, + /// Recent measured render GPU time per pass by VM. Smoothed over batches; + /// replacing the least recently measured VM does not retain its mappings. + pass_cost: policy::PassCosts, + /// Latched only after the isolated InitBM draw fully retired. + render_initialized: bool, + config: Config, + uat: mmu::Uat, + drm: driver::AsahiDevRef, + device: Device, +} +impl Inner { + /// Latch before allocating diagnostics, once per runtime. Called under + /// its mutex even for errors outside the active-job polling loop. + fn capture_fault(&mut self, primary: Error) { + self.state.health.mark_failed(); + if self.fault_captured { return; } + self.fault_captured = true; + #[cfg(CONFIG_DEV_COREDUMP)] + if let Err(error) = self.config.fault_snapshot(&self.drm, primary, self.gpu_pending) { + dev_err!(self.drm.as_ref(), "M3 fault snapshot failed: {:?}\n", error); + } + } + + /// Record a batch that failed or never retired. Mark the GPU failed first, + /// so that a failing diagnostic read cannot leave a job that never retired + /// behind a device that still reports itself healthy. + fn fail(&mut self, index: usize, vm: &mmu::Vm, primary: Error) { + self.capture_fault(primary); + if crate::t8122_start::is_t8122(self.device.soc()) { + let pstate=self.device.pstate_register().ok(); + // The job's first GPU start timestamp: nonzero means the firmware dispatched it, + // whatever the power state reads at the timeout. + let start_stamp=self.jobs[index].batch_gpu_span().ok().map(|span| span[0]); + crate::t8122_start::job_failed_verdict(self.drm.as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::job_failed_verdict(self.drm.as_ref(), primary, self.state.health.crashed()); + self.t8122_verdict=true; + } + let events=self.state.event_messages.load(Ordering::Acquire); + if let Err(e) = self.jobs[index].log(&self.drm) { + dev_err!(self.drm.as_ref(), "M3 job diagnostics failed: {:?}\n", e); + } + if let Err(e) = self.device.log_engine_state(vm) { + dev_err!(self.drm.as_ref(), "M3 engine diagnostics failed: {:?}\n", e); + } + let _=self.config.log_recovery_state(&self.drm,events); + if crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault) { + if let Err(e)=self.resume_experiment(index, vm) { + dev_err!(self.drm.as_ref(),"M3: resume-after-fault: stopped by {:?}\n",e); + } + } + } + /// asahi.g15_debug bit 56. Wait up to 100 ms for the firmware to report + /// that it halted, resume it, and log every change of the halt words, + /// the engine state, the failed job's retirement, whether every queue + /// message was consumed and the event count for about half a second, + /// then the event ring entries the host has not consumed. The GPU stays + /// marked failed; the event ring's read index is not advanced. + fn resume_experiment(&mut self, index: usize, vm: &mmu::Vm) -> Result { + let start=Instant::::now(); + let mut halt=self.config.halt_state()?; + while halt.1==0 && start.elapsed()::now(); + let mut last=None; + for _ in 0..500 { + let halt=self.config.halt_state()?; + // 0: idle and no fault bits, 1: an engine is busy, 2: fault bits set. + let engines=match self.device.check_idle() {Ok(())=>0u8,Err(e) if e==EBUSY=>1,Err(_)=>2}; + let complete=self.jobs[index].complete().unwrap_or(false); + // 0: queues not drained, 1: every submitted queue message consumed, 2: read failed. + let pipes=match self.config.pipes_idle() {Ok(false)=>0u8,Ok(true)=>1,Err(_)=>2}; + let events=self.state.event_messages.load(Ordering::Acquire); + let crashed=self.state.health.crashed(); + let now=(halt,engines,complete,pipes,events,crashed); + if last!=Some(now) { + dev_info!(self.drm.as_ref(),"M3: resume-after-fault: +{}us: halt_count={} halted={} resume={} engines={} job_complete={} pipes_idle={} event_messages={} crashed={}\n", + resumed.elapsed().as_nanos()/1000,halt.0,halt.1,halt.2,engines,complete,pipes,events,crashed); + last=Some(now); + } + if crashed {break;} + fsleep(Delta::from_millis(1)); + } + self.jobs[index].log(&self.drm)?; + self.device.log_engine_state(vm)?; + self.config.log_event_ring(&self.drm)?; + let events=self.state.event_messages.load(Ordering::Acquire); + self.config.log_recovery_state(&self.drm,events) + } +} +impl Inner { + /// Fail every packet that was never published. Their fences signal with the error. + fn fail_pending(&mut self, error: Error) { + for (packet,_) in self.pending.drain_all() {packet.finish(Err(error));} + } + /// A published batch failed or never retired: latch the fault, log it, and fail its + /// packets. The packets stay retained (with their VM job guards) until teardown, because + /// the firmware has not provably stopped using their memory. + fn fail_active(&mut self, error: Error) { + let _=self.config.performance_request(self.device.soc().chip_id,false); + if let Some((index,vm))=self.active.first().map(|b|(b.index,b.entries[0].0.vm.clone())) { + self.log_owners(error); + self.fail(index,&vm,error); + } else {self.capture_fault(error);} + for batch in self.active.drain_all() {Self::finish_all(&batch.entries,Err(error));} + self.fail_pending(error); + } + /// Name the processes whose published commands were in flight when the GPU failed. Batches + /// in flight share one VM, so this is normally one process; a VM shared between processes + /// names each submitter once (the first four). + fn log_owners(&self, error: Error) { + let commands:usize=self.active.iter().map(|b|b.entries.len()).sum(); + let mut logged=[0i32;4]; + let mut count=0; + for (packet,_) in self.active.iter().flat_map(|b|b.entries.iter()) { + let owner=&packet.owner; + if logged[..count].contains(&owner.pid) {continue;} + if count==logged.len() {break;} + dev_err!(self.drm.as_ref(),"M3 failed work owner: VM={} process {} pid {} thread {} (error {:?}, {} commands in flight)\n", + packet.vm.id(),owner.process(),owner.pid,owner.thread(),error,commands); + logged[count]=owner.pid;count+=1; + } + } + /// Hardware completion is separate from fence signaling: cancellation can win the + /// signal race while the DMA guard must survive until its final published batch retires. + fn packet_active(&self, packet:&Arc)->bool { + self.active.iter().any(|b|b.entries.iter().any(|(p,_)|Arc::ptr_eq(p,packet))) + } + fn retire_packets(&mut self,entries:&[(Arc,usize)]) { + for (packet,_) in entries { + let pending=self.pending.iter().any(|(p,_)|Arc::ptr_eq(p,packet)); + if policy::packet_retired(pending,self.packet_active(packet)) {packet.retired();} + } + // Keep exactly active DMA owners. Successful retirement may precede a VM switch. + self.packets.retain(|p| self.active.iter().any(|b|b.entries.iter().any(|(q,_)|Arc::ptr_eq(p,q)))); + } + fn fail_front(&mut self,error:Error) { + if let Ok((packet,_))=self.pending.remove(0) { + if !self.packet_active(&packet) && !self.fault_captured {packet.release_guard();} + packet.finish(Err(error)); + } + } + /// Signal every packet of `entries` (packets that continue past the batch included). + fn finish_all(entries:&[(Arc,usize)],result:Result) { + for (packet,_) in entries {packet.finish(result);} + } + /// Choose the next batch from the front of `pending`: the front packet's next commands of + /// one engine, and for render also the leading render commands of following packets of the + /// same VM, up to the ordered-batch limit. + fn next_batch(&self,max:usize)->Result,usize)>> { + let mut entries=KVec::new(); + let Some((packet,next))=self.pending.first() else {return Ok(entries);}; + let mixed=policy::mixed_prefix(crate::m3_params::mixed_batch(), + self.device.soc().chip_id,self.render_initialized,self.active.is_empty(), + packet.commands[*next..].iter().map(|c|matches!(c,crate::m3_submit::Command::Render{..}))); + if mixed>0 { + for index in *next..*next+mixed {entries.push((packet.clone(),index),GFP_KERNEL)?;} + return Ok(entries); + } + let render=matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}); + let limit=if render { + // The first render draw initializes the shared buffer manager (InitBM) and opens + // both queues. Publish it alone: appending later draws first advances the shared + // manager and event counters it starts from, and on J613 the second pass's + // fragment then fails with firmware error event 4 (draw=2 batch=2). + let started=self.jobs.iter().any(|job| matches!(job,NativeJob::Render(j) if j.started())); + if started {crate::m3_params::render_batch_size().min(max).max(1)} else {1} + } else { + crate::m3_params::compute_batch_size() + .clamp(1,crate::m3_compute_storage::SLOTS) + }; + let same_engine=|c:&crate::m3_submit::Command| render==matches!(c,crate::m3_submit::Command::Render{..}); + let per_pass=self.pass_cost.get(packet.vm.id()); + let budget=crate::m3_params::render_batch_budget_ns(self.device.soc()); + let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); + for (position,(candidate,start)) in self.pending.iter().enumerate() { + // Later packets join only render batches, only from their first command, and only + // for the VM the batch is bound to, while the batch's expected GPU time is within + // the budget. + if position>0 && !policy::may_join(render,*start,candidate.vm.same(&packet.vm),candidate.urgency==packet.urgency) {break;} + let mut index=*start; + while index=limit || indexResult> { + let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3_pass_layout::SLOTS)));}; + if !self.render_initialized {return Ok(None);} + let Some((packet,next))=self.pending.first() else {return Ok(None);}; + if !matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}) + || self.active.iter().any(|b| b.kind!=0) || !packet.vm.same(&last.entries[0].0.vm) {return Ok(None);} + let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); + let Some(room)=policy::pipeline_room(crate::m3_params::pipeline_depth(self.device.soc()), + self.active.len(),in_flight,last.base,last.entries.len()) else {return Ok(None);}; + // Yield at the next retirement when more urgent work is already waiting. + if policy::urgent_waiting(self.pending.iter().map(|(p,_)|p.urgency), + last.entries[0].0.urgency) {return Ok(None);} + if !policy::within_budget(self.pass_cost.get(packet.vm.id()), + crate::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} + if !policy::overlap_ready(self.render_initialized,true,true,true, + self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);} + Ok(Some(room)) + } + /// Publish pending batches while allowed: one when none is active, more behind it with + /// asahi.m3_pipeline_depth > 1. Returns whether anything was published or failed. + fn publish_next(&mut self)->bool { + let mut progress=false; + while !self.pending.is_empty() && self.state.healthy() { + let overlap=!self.active.is_empty(); + let (base,max)=match self.pipeline_slot() { + Ok(Some(slot))=>slot, + Ok(None)=>break, + Err(error)=>{self.fail_active(error);return true;} + }; + let entries=match self.next_batch(max) { + Ok(entries) if !entries.is_empty()=>entries, + _=>{self.fail_front(ENOMEM);progress=true;continue;}, + }; + match self.publish(&entries,base,overlap) { + Ok(batch)=>{ + // Consume the published commands from `pending`. + let Some((last,last_index))=entries.last().map(|(p,i)|(p.clone(),*i)) else {break;}; + while let Some((packet,_))=self.pending.first() { + if Arc::ptr_eq(packet,&last) { + if last_index+1==last.commands.len() {let _=self.pending.remove(0);} + else {self.pending[0].1=last_index+1;} + break; + } + let _=self.pending.remove(0); + } + if self.active.push(batch,GFP_KERNEL).is_err() { + // Published but untracked: it can never be retired. + self.capture_fault(ENOMEM);Self::finish_all(&entries,Err(ENOMEM)); + self.fail_active(ENOMEM);return true; + } + progress=true; + } + Err((error,published))=>{ + if published { + if crate::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { + crate::t8122_start::job_setup_failed_verdict(self.drm.as_ref(),error); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) && !self.t8122_verdict { + crate::t6031_start::job_setup_failed_verdict(self.drm.as_ref(), error); + self.t8122_verdict=true; + } + // A live ring WRITE or doorbell may have been published: keep packets + // retained (self.packets) and fail them and everything in flight. + self.capture_fault(error); + Self::finish_all(&entries,Err(error)); + self.fail_active(error); + self.fail_pending(error); + return true; + } else if overlap && error==EBUSY { + // Not publishable behind the running batch (new VM binding or storage + // growth): wait for it to retire. + break; + } else { + // Nothing reached the firmware: fail only the front packet, as a failed + // execute() did before publication. + self.fail_front(error); + progress=true; + } + } + } + } + progress + } + /// Prepare and publish one batch. On error, reports whether firmware may see its writes. + fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { + let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Render{..})); + let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Compute(_))); + if render && compute {return self.publish_mixed(entries,base,overlap);} + let early=|e:Error|(e,false); + let (packet,first_index)=(&entries[0].0,entries[0].1); + let control=packet.commands[first_index]; + let batch_count=entries.len(); + let prepare=Instant::::now(); + // All host bookkeeping is reserved before mutating shared producer state. + // No allocation may fail after publication and lose a DMA owner. + self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.packets.reserve(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut owned=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + for (p,i) in entries {owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + + // Retain the batch's packets (and their VM job guards) until it retires; with batches + // in flight, also those of the earlier ones. + if !overlap {self.packets.clear();} + for (p,_) in entries { + if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + } + let found=self.jobs.iter().position(|job| matches!((job,control), + (NativeJob::Compute(_),crate::m3_submit::Command::Compute(_)) | + (NativeJob::Render(_),crate::m3_submit::Command::Render{..}))); + let index=found.unwrap_or(self.jobs.len()); + if found.is_none() { + let job=match control { + crate::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + }; + self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + } else { + match (&mut self.jobs[index],control) { + (NativeJob::Render(_),crate::m3_submit::Command::Render{..})=>{}, + (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, + _=>return Err(early(ENOTSUPP)), + } + } + if let NativeJob::Compute(j)=&mut self.jobs[index] { + let prepared=(||->Result { + for (slot,(p,next)) in entries.iter().enumerate() { + if slot!=0 { + let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + j.append(c)?; + } + let timestamps=p.timestamp_addresses(*next); + let coalesce=crate::m3_compute_storage::coalesce_stamp_flush( + p.wide_visibility[*next],timestamps,slot+1==batch_count); + j.prepare_completion(coalesce)?; + j.set_user_timestamps(timestamps)?; + j.set_attachments(&p.attachments[*next])?; + } + Ok(()) + })(); + if let Err(error)=prepared { + j.abort_batch().map_err(|e|(e,true))?; + if found.is_none() {drop(self.jobs.remove(index).map_err(|_|early(EIO))?);} + return Err(early(error)); + } + } + let mut coalesced_stamps=0; + let coalesce_render=crate::m3_params::render_stamp_coalesce(); + let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let chip=self.device.soc().chip_id; + if let NativeJob::Render(j)=&mut self.jobs[index] { + let mut commands=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + for (p,next) in entries {commands.push(p.commands[*next],GFP_KERNEL).map_err(|_|early(ENOMEM))?;} + j.begin_batch(&self.drm,&self.uat,&packet.vm,&commands,base,overlap).map_err(early)?; + let prepared=(||->Result { + for (slot,(p,next)) in entries.iter().enumerate() { + let crate::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; + let timestamps=p.render_timestamp_addresses(*next); + let coalesce=policy::coalesce_render_stamps(coalesce_render,chip, + overlap,slot,batch_count,timestamps); + let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,chip,pair)); + j.append(command,usc,coalesce,timed)?; + j.set_user_timestamps(timestamps)?; + coalesced_stamps+=u64::from(coalesce); + } + Ok(()) + })(); + if let Err(error)=prepared { + // Running firmware can consume append's WRITE updates without a new doorbell. + // Even an append that returns an error may have changed shared counters or one + // pipe. Do not rewind live producer state or let the caller release owners and + // reuse these slots. Its published-error path stops scheduling and retains them. + if overlap {return Err((error,true));} + j.abort_batch().map_err(|e|(e,true))?; + return Err(early(error)); + } + } + let preparation_ns=prepare.elapsed().as_nanos(); + self.t8122_verdict=false; + let previous_events=self.config.completed_events; + let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; + self.state.health.set_gpu_pending(true); + self.gpu_pending = true; + let late=|e:Error|(e,true); + let expected_events=match &self.jobs[index] { + NativeJob::Compute(j)=>{ + self.config.submit_queue(2,j.queue(),j.head(),2,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000002).map_err(late)?;1 + }, + NativeJob::Render(j)=>{ + if j.first() {self.config.render_pb().map_err(late)?;} + let queues=j.queues(); + self.config.submit_queue(1,queues[1],j.heads()[1],1,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000001).map_err(late)?; + self.config.submit_queue(0,queues[0],j.heads()[0],0,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000000).map_err(late)?;2 + }, + }; + let measure=crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary); + let (base,last_draw)=match &self.jobs[index] {NativeJob::Render(j)=>{let (b,_,d)=j.batch();(b,d)},_=>(0,0)}; + Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3_submit::Command::Compute(_))), + first:control, preparation_ns, previous_events, expected_events, + start:Instant::::now(), measure, doorbell_counter_ns:if measure {physical_counter() as i64 * 1000 / 24} else {0}, + polling_ns:0, polls:0, stamp_ns:-1, first_event_ns:-1, events_ns:-1, completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0, coalesced_stamps, base, last_draw, publication_tick, mixed:None }) + } + /// Prepare both idle queues from one packet. Cross-engine changes wait on + /// the preceding command's firmware stamp; all commands remain GPU ordered. + /// Producer storage and DMA guards survive every partial publication error. + fn publish_mixed(&mut self,entries:&[(Arc,usize)], + base:usize,overlap:bool)->core::result::Result { + let early=|e:Error|(e,false); + if overlap || base!=0 || self.device.soc().chip_id!=0x6030 + || !self.render_initialized || entries.len()>policy::SLOTS {return Err(early(EINVAL));} + let packet=&entries[0].0; + if entries.iter().any(|(p,_)|!Arc::ptr_eq(packet,p)) {return Err(early(EINVAL));} + let prepare=Instant::::now(); + let internal_timestamps=crate::m3_params::render_internal_timestamps(); + self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.jobs.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + self.packets.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut owned=KVec::with_capacity(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut render_commands=KVec::with_capacity(entries.len(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut first_compute=None; + let mut computes=0; + for (p,i) in entries { + owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + match p.commands[*i] { + c@crate::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, + crate::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, + } + } + let renders=render_commands.len(); + if renders==0 || computes==0 {return Err(early(EINVAL));} + let index=self.jobs.iter().position(|j|matches!(j,NativeJob::Render(r) if r.started())).ok_or(early(EINVAL))?; + let existing=self.jobs.iter().position(|j|matches!(j,NativeJob::Compute(_))); + let compute_index=existing.unwrap_or(self.jobs.len()); + // Keep the packet before any producer mutation. No other batch is live. + self.packets.clear(); + self.packets.push(packet.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?; + let mut render_prepared=false; + let mut compute_prepared=false; + let prepared=(||->Result { + let control=first_compute.ok_or(EINVAL)?; + if existing.is_none() { + let job=crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, + self.config.stats_region()?,control,self.device.soc().registers)?; + self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; + } else { + let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; + j.replay(&self.uat,&packet.vm,control)?; + } + compute_prepared=true; + if let NativeJob::Compute(j)=&mut self.jobs[compute_index] { + j.reserve_barriers(&self.drm,&self.uat)?; + } + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; + j.begin_batch(&self.drm,&self.uat,&packet.vm,&render_commands,0,false)?; + render_prepared=true; + let mut previous:Option<(bool,crate::m3_sync_layout::CompletionDependency)>=None; + let mut compute_slot=0; + let mut dependencies=0; + for (p,i) in entries { + match p.commands[*i] { + crate::m3_submit::Command::Render{command,usc}=>{ + let predecessor=previous.filter(|(render,_)|!*render).map(|(_,d)|d); + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; + let timestamps=p.render_timestamp_addresses(*i); + let timed=timestamps.map(|pair|policy::stage_timestamps(internal_timestamps,self.device.soc().chip_id,pair)); + j.append_after(command,usc,false,predecessor,timed)?; + j.set_user_timestamps(timestamps)?; + dependencies+=usize::from(predecessor.is_some()); + previous=Some((true,j.completion_dependency()?)); + }, + crate::m3_submit::Command::Compute(c)=>{ + let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; + if compute_slot!=0 {j.append(c)?;} + if let Some((true,dependency))=previous { + j.insert_dependency(dependency)?; + dependencies+=1; + } + // Every cross-engine predecessor flushes its stamp. + // Final queue, timestamp and cache visibility are unchanged. + j.prepare_completion(false)?; + j.set_user_timestamps(p.timestamp_addresses(*i))?; + j.set_attachments(&p.attachments[*i])?; + previous=Some((false,j.completion_dependency()?)); + compute_slot+=1; + }, + } + } + Ok(dependencies) + })(); + let dependencies=match prepared { + Ok(n)=>n, + Err(error)=>{ + if render_prepared { + let NativeJob::Render(j)=&mut self.jobs[index] else {return Err((EIO,true));}; + j.abort_batch().map_err(|e|(e,true))?; + } + if compute_prepared { + if existing.is_none() {drop(self.jobs.remove(compute_index).map_err(|_|(EIO,true))?);} + else if let NativeJob::Compute(j)=&mut self.jobs[compute_index] { + j.abort_batch().map_err(|e|(e,true))?; + } + } + return Err(early(error)); + }, + }; + let preparation_ns=prepare.elapsed().as_nanos(); + let previous_events=self.config.completed_events; + let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; + self.state.health.set_gpu_pending(true);self.gpu_pending=true; + let late=|e:Error|(e,true); + let NativeJob::Render(j)=&self.jobs[index] else {return Err(late(EIO));}; + let queues=j.queues();let heads=j.heads();let (_,_,last_draw)=j.batch(); + self.config.submit_queue(1,queues[1],heads[1],1,false).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000001).map_err(late)?; + self.config.submit_queue(0,queues[0],heads[0],0,false).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000000).map_err(late)?; + let NativeJob::Compute(j)=&self.jobs[compute_index] else {return Err(late(EIO));}; + self.config.submit_queue(2,j.queue(),j.head(),2,j.first()).map_err(late)?; + Pin::new(&mut self.transport).send_message(0x21,0x0083000000000002).map_err(late)?; + Ok(Batch {entries:owned,index,kind:2,first:packet.commands[entries[0].1], + preparation_ns,previous_events,expected_events:3,start:Instant::::now(), + measure:crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary), + doorbell_counter_ns:0,polling_ns:0,polls:0,stamp_ns:-1,first_event_ns:-1, + events_ns:-1,completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0,coalesced_stamps:0,base:0,last_draw,publication_tick, + mixed:Some(MixedBatch {compute_index,renders,computes,dependencies})}) + } + /// A mixed publication retires only after both jobs, all three queue + /// messages, firmware health and engine idle passed the ordinary checks. + /// Its wall time is reported once rather than charged to both engines. + fn retire_mixed(&mut self,batch:&Batch,mixed:&MixedBatch)->Result { + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let render_ns=j.gpu_ns(0,mixed.renders)?; + let stages=j.stage_ns(0,mixed.renders)?; + j.progress(&self.drm)?; + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let compute_ns=j.gpu_ns()?; + j.progress(&self.drm)?; + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let vm=batch.entries[0].0.vm.id(); + let observed_tick=physical_counter(); + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let render_span=j.gpu_ticks(0,mixed.renders)?; + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let compute_span=j.batch_gpu_span()?; + let start=render_span.0.min(compute_span[0]); + let end=render_span.1.max(compute_span[1]); + let valid=batch.publication_tick!=0 && start>=batch.publication_tick && end>=start && observed_tick>=end; + dev_info!(self.drm.as_ref(),"M3_BATCH_BOUNDARY vm={} mixed=1 batch={} publication_tick={} gpu_start_tick={} gpu_end_tick={} observed_tick={} counter_hz=24000000 valid={}\n", + vm,batch.entries.len(),batch.publication_tick,start,end,observed_tick,u32::from(valid)); + self.log_batch_milestones(batch,render_span.0,render_span.1,render_ns,stages[1]); + // Render and compute have separate slot sequences in one mixed batch. + let mut render_slot=0; + let mut compute_slot=0; + for (packet,next) in batch.entries.iter() { + match packet.commands[*next] { + crate::m3_submit::Command::Render{command:r,..}=>{ + let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; + let s=j.slot_stage_ns(0,mixed.renders,render_slot)?; + dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} mixed=1 slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n", + vm,render_slot,mixed.renders,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); + render_slot+=1; + }, + crate::m3_submit::Command::Compute(c)=>{ + let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; + let ns=j.slot_gpu_ns(compute_slot)?; + dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} mixed=1 slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n", + vm,compute_slot,mixed.computes,c.base,c.end,c.usc_base,ns); + compute_slot+=1; + }, + } + } + } + self.state.health.record_completion(); + self.render_batches[mixed.renders-1]+=1; + self.pass_cost.record(batch.entries[0].0.vm.id(),render_ns/mixed.renders as u64); + self.mixed_batches+=1; + if (self.mixed_batches==1 || self.mixed_batches%128==0) && batch.measure { + dev_info!(self.drm.as_ref(),"M3_MIXED_BATCH_RETIRED count={} commands={} render={} compute={} barriers={} prepare_ns={} active_ns={} render_gpu_ns={} compute_gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} stamps=1 queues=1 events=3 idle=1\n", + self.mixed_batches,batch.entries.len(),mixed.renders,mixed.computes,mixed.dependencies, + batch.preparation_ns,batch.start.elapsed().as_nanos(),render_ns,compute_ns,stages[0],stages[1],stages[2]); + } + if self.active.is_empty() { + self.gpu_pending=false;self.state.health.set_gpu_pending(false); + } + Ok(()) + } + /// Check the active batch once. Ok(true): retired (its packets are signalled). + fn poll_active(&mut self)->Result { + let Some((index,measure,last_draw))=self.active.first().map(|b|(b.index,b.measure,b.last_draw)) else {return Ok(false);}; + let poll_start=measure.then(Instant::::now); + if !self.state.healthy() || self.config.drain(&self.drm).is_err() { + self.fail_active(EIO);return Err(EIO); + } + // With later batches in flight, the oldest retires once its own last draw's stamps + // have arrived; the last one is also checked for exact queue, event and engine idle. + if self.active.len()>1 { + let done=match &mut self.jobs[index] {NativeJob::Render(j)=>j.complete_through(last_draw),_=>Err(EINVAL)}; + match done { + Ok(true)=>{ + let batch=self.active.remove(0).map_err(|_|EIO)?; + if let Err(e)=self.retire(&batch) { + self.capture_fault(e);Self::finish_all(&batch.entries,Err(e)); + self.fail_active(e);return Err(e); + } + self.retire_packets(&batch.entries); + return Ok(true); + } + Ok(false)=>{ + let (expired,previous)=self.active.first().map(|b|(b.start.elapsed()>=Delta::from_secs(2),b.previous_events)).ok_or(EIO)?; + if expired { + dev_err!(self.drm.as_ref(),"M3 pipelined completion events={} previous={}\n",self.config.completed_events,previous); + self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT); + } + return Ok(false); + } + Err(e)=>{self.fail_active(e);return Err(e);} + } + } + let secondary=self.active.first().and_then(|b|b.mixed.as_ref().map(|m|m.compute_index)); + let completion=(||->Result { + let primary=self.jobs[index].complete()?; + let other=match secondary {Some(i)=>self.jobs[i].complete()?,None=>true}; + Ok(primary && other) + })(); + let stamped=match completion {Ok(v)=>v,Err(e)=>{self.fail_active(e);return Err(e);}}; + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let before=physical_counter(); + let parts=match &mut self.jobs[index] { + NativeJob::Render(j)=>j.fragment_stamp_parts().ok(), + _=>None, + }; + let after=physical_counter(); + if let Some(parts)=parts { + if let Some(batch)=self.active.first_mut() { + if batch.fragment_host_tick==0 { + if parts[0] {batch.fragment_host_tick=after;} + else {batch.fragment_host_pending_tick=before;} + } + if batch.fragment_fw_tick==0 { + if parts[1] {batch.fragment_fw_tick=after;} + else {batch.fragment_fw_pending_tick=before;} + } + } + } + } + let power_enabled=crate::m3_params::active_power(); + if power_enabled { + let since_last=self.active_power_last.map(|last|last.elapsed().as_nanos()); + if crate::m3_active_power::due(power_enabled,self.device.soc().chip_id, + stamped,self.active_power_samples,since_last) { + self.active_power_last=Some(Instant::::now()); + self.active_power_samples+=1; + // Diagnostic read errors do not change job completion or recovery. + if let Err(e)=self.config.log_active_power(&self.drm,&self.device, + self.active_power_samples) { + dev_info!(self.drm.as_ref(),"M3_ACTIVE_POWER sample={} unavailable={:?}\n", + self.active_power_samples,e); + } + } + } + let completion_parts=if measure && secondary.is_none() { + match &mut self.jobs[index] { + NativeJob::Render(j)=>Some(if stamped {[true;4]} else { + match j.completion_parts() {Ok(v)=>v,Err(e)=>{self.fail_active(e);return Err(e);}} + }), + _=>None, + } + } else {None}; + let completed=self.config.completed_events; + let (evented,expired,previous)={ + let batch=self.active.first_mut().ok_or(EIO)?; + let evented=completed>=batch.previous_events+batch.expected_events; + if batch.measure { + let at=batch.start.elapsed().as_nanos(); + if stamped && batch.stamp_ns<0 {batch.stamp_ns=at;} + if let Some(parts)=completion_parts { + for (part,done) in parts.iter().enumerate() { + if *done && batch.completion_ns[part]<0 {batch.completion_ns[part]=at;} + } + } + if completed>batch.previous_events && batch.first_event_ns<0 {batch.first_event_ns=at;} + if evented && batch.events_ns<0 {batch.events_ns=at;} + } + if let Some(t)=poll_start {batch.polling_ns+=t.elapsed().as_nanos();batch.polls+=1;} + (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events) + }; + if stamped && evented { + let retire_mmio=crate::m3_params::retire_mmio(self.device.soc()); + match self.device.check_idle_parts( + retire_mmio&crate::m3_params::RETIRE_MMIO_BUSY!=0, + retire_mmio&crate::m3_params::RETIRE_MMIO_FAULTS!=0) { + Ok(())=>match self.config.pipes_idle() { + Ok(true)=>{ + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let tick=physical_counter(); + if let Some(batch)=self.active.first_mut() {batch.pipes_idle_tick=tick;} + } + if retire_mmio&crate::m3_params::RETIRE_MMIO_PSTATE!=0 { + if let Err(e)=self.config.check_pstate(&self.drm,&self.device,"after a job") { + if crate::t8122_start::is_t8122(self.device.soc()) { + crate::t8122_start::cap_violated_verdict(self.drm.as_ref(),e); + self.t8122_verdict=true; + } else if crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::cap_violated_verdict(self.drm.as_ref(), e); + self.t8122_verdict=true; + } + self.fail_active(e);return Err(e); + } + } + // Stamps, both queue indices, required events, + // firmware health, engines and pipes are verified. + let batch=self.active.remove(0).map_err(|_|EIO)?; + if let Err(e)=self.retire(&batch) { + self.capture_fault(e); + Self::finish_all(&batch.entries,Err(e)); + self.fail_pending(e); + return Err(e); + } + // Signal every packet whose last command was in this batch. A packet + // that continues with other commands is still at the front of `pending`. + self.retire_packets(&batch.entries); + return Ok(true); + }, + Ok(false)=>{}, // Firmware must consume every submitted queue message. + Err(e)=>{self.fail_active(e);return Err(e);} + }, + Err(e) if e==EBUSY=>{}, // Retirement can trail the event. + Err(e)=>{self.fail_active(e);return Err(e);} + } + } + if expired { + dev_err!(self.drm.as_ref(),"M3 completion events={} previous={}\n",completed,previous); + self.fail_active(ETIMEDOUT);return Err(ETIMEDOUT); + } + Ok(false) + } + fn log_batch_milestones(&self,batch:&Batch,render_start:u64,render_end:u64, + render_gpu_ns:u64,fragment_sum_ns:u64) { + dev_info!(self.drm.as_ref(),"M3_BATCH_MILESTONE vm={} publication_tick={} draw={} mixed={} render_start_tick={} render_end_tick={} render_gpu_ns={} fragment_sum_ns={} fragment_host_tick={} fragment_fw_tick={} fragment_host_pending_tick={} fragment_fw_pending_tick={} pipes_idle_tick={} counter_hz=24000000\n", + batch.entries[0].0.vm.id(),batch.publication_tick,batch.last_draw,u32::from(batch.mixed.is_some()), + render_start,render_end,render_gpu_ns,fragment_sum_ns,batch.fragment_host_tick,batch.fragment_fw_tick, + batch.fragment_host_pending_tick,batch.fragment_fw_pending_tick,batch.pipes_idle_tick); + for (packet,command) in batch.entries.iter() { + dev_info!(self.drm.as_ref(),"M3_BATCH_PACKET vm={} publication_tick={} packet={:#x} command={}\n", + packet.vm.id(),batch.publication_tick,packet.profile_id(),command); + } + } + /// Bookkeeping for a retired batch, then signal its completed packets. + fn retire(&mut self,batch:&Batch)->Result { + if let Some(mixed)=&batch.mixed {return self.retire_mixed(batch,mixed);} + let retire_start=batch.measure.then(Instant::::now); + let index=batch.index;let batch_count=batch.entries.len();let control=batch.first; + self.state.health.record_completion(); + if self.active.is_empty() { + match &mut self.jobs[index] { + NativeJob::Compute(j)=>j.progress(&self.drm)?, + NativeJob::Render(j)=>j.progress(&self.drm)?, + } + } + let retired_counter_ns=if batch.measure {physical_counter() as i64 * 1000 / 24} else {0}; + let ticks=match &mut self.jobs[index] { + NativeJob::Render(j) if batch.measure=>Some(j.gpu_ticks(batch.base,batch_count)?), + _=>None, + }; + let stages=match &mut self.jobs[index] { + NativeJob::Render(j)=>j.stage_ns(batch.base,batch_count)?, + _=>[0;3], + }; + let (kind,gpu_ns)=match &mut self.jobs[index] { + NativeJob::Compute(j)=>(1,j.gpu_ns()?), + NativeJob::Render(j)=>(0,j.gpu_ns(batch.base,batch_count)?), + }; + if kind==0 { + self.render_batches[batch_count-1]+=1; + if batch.measure && batch.coalesced_stamps>0 && self.coalesced_render_stamps==0 { + if let NativeJob::Render(j)=&mut self.jobs[index] { + let masks=j.stamp_flush_masks(batch.base,batch_count)?; + dev_info!(self.drm.as_ref(),"M3_RENDER_STAMP_FLUSH vm={} batch={} ta_mask={:#x} fs_mask={:#x} coalesced={}\n", + batch.entries[0].0.vm.id(),batch_count,masks[0],masks[1],batch.coalesced_stamps); + } + } + self.coalesced_render_stamps+=batch.coalesced_stamps; + let vm=batch.entries[0].0.vm.id(); + self.pass_cost.record(vm,gpu_ns/batch_count as u64); + self.render_initialized=true; + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { + let observed_tick=physical_counter(); + let (cs,span)=match (&mut self.jobs[index],control) { + (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), + (NativeJob::Render(j),crate::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, + _=>return Err(EINVAL), + }; + let valid=batch.publication_tick!=0 && span[0]>=batch.publication_tick && span[1]>=span[0] && observed_tick>=span[1]; + dev_info!(self.drm.as_ref(),"M3_BATCH_BOUNDARY vm={} cs={:#x} batch={} publication_tick={} gpu_start_tick={} gpu_end_tick={} observed_tick={} counter_hz=24000000 valid={}\n", + batch.entries[0].0.vm.id(),cs,batch_count,batch.publication_tick,span[0],span[1],observed_tick,u32::from(valid)); + self.log_batch_milestones(batch, + if kind==0 {span[0]} else {0},if kind==0 {span[1]} else {0}, + if kind==0 {gpu_ns} else {0},stages[1]); + if let NativeJob::Compute(j)=&mut self.jobs[index] { + for (slot,(p,next)) in batch.entries.iter().enumerate() { + let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let ns=j.slot_gpu_ns(slot)?; + dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n",p.vm.id(),slot,batch_count,c.base,c.end,c.usc_base,ns); + } + } + if let NativeJob::Render(j)=&mut self.jobs[index] { + for (slot,(p,next)) in batch.entries.iter().enumerate() { + let crate::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; + let s=j.slot_stage_ns(batch.base,batch_count,slot)?; + dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n",p.vm.id(),slot,batch_count,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); + } + } + } + let preparation_ns=batch.preparation_ns; + let active_ns=batch.start.elapsed().as_nanos(); + self.state.health.timing.record(kind==0, batch_count as u64, + preparation_ns, active_ns, gpu_ns, stages); + let retirement_ns=retire_start.map_or(0,|t|t.elapsed().as_nanos()); + if batch.measure { + if let crate::m3_submit::Command::Render {command:r,..}=control { + if batch_count>1 {self.geometry_overflow+=1;} else { + if let Some(g)=self.geometry.iter_mut().find(|g| + g.count==0 || (g.width==r.width && g.height==r.height && g.samples==r.samples)) { + g.width=r.width;g.height=r.height;g.samples=r.samples; + g.count+=1;g.prepare_ns+=preparation_ns;g.active_ns+=active_ns; + g.gpu_ns+=gpu_ns;g.tiling_ns+=stages[0];g.fragment_ns+=stages[1]; + g.max_fragment_ns=g.max_fragment_ns.max(stages[1]); + g.polling_ns+=batch.polling_ns;g.retirement_ns+=retirement_ns; + g.polls+=batch.polls; + g.stamp_ns+=batch.stamp_ns;g.first_event_ns+=batch.first_event_ns;g.events_ns+=batch.events_ns; + for (total,observed) in g.completion_ns.iter_mut().zip(batch.completion_ns) {*total+=observed;} + if let Some((ta,fe))=ticks { + g.pickup_ns+=(ta as i64)*1000/24-batch.doorbell_counter_ns; + g.tail_ns+=retired_counter_ns-(fe as i64)*1000/24; + } + } else { + self.geometry_overflow+=1; + } + } + } + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + match control { + crate::m3_submit::Command::Render {command:r,..}=> + dev_info!(self.drm.as_ref(),"M3_PASS kind=render batch={} vdm={:#x} width={} height={} samples={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={}\n",batch_count,r.vdm_base,r.width,r.height,r.samples,preparation_ns,active_ns,gpu_ns,stages[0],stages[1],stages[2]), + crate::m3_submit::Command::Compute(_)=> + dev_info!(self.drm.as_ref(),"M3_PASS kind=compute prepare_ns={} active_ns={} gpu_ns={}\n",preparation_ns,active_ns,gpu_ns), + } + } + let t=&mut self.timing[kind]; + t[0]+=1;t[1]+=preparation_ns;t[2]+=active_ns;t[3]+=gpu_ns as i64; + for i in 0..3 {t[4+i]+=stages[i] as i64;} + // Running totals are in debugfs `timing`; the periodic line is a SubmitTiming diagnostic. + if t[0]==1 && crate::t8122_start::is_t8122(self.device.soc()) { + let span=match &mut self.jobs[index] { + NativeJob::Compute(j)=>j.batch_gpu_span().ok(), + NativeJob::Render(j)=>j.gpu_ticks(batch.base,batch_count).ok().map(|(a,b)|[a,b]), + }; + crate::t8122_start::job_completed_verdict(self.drm.as_ref(),kind,gpu_ns,span); + } else if t[0]==1 && crate::t6031_start::is_t6031(self.device.soc()) { + crate::t6031_start::job_completed_verdict(self.drm.as_ref(), kind, gpu_ns); + } + if t[0]%128==0 && crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { + dev_info!(self.drm.as_ref(),"M3_TIMING kind={} count={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} ordinal={}\n",kind,t[0],t[1],t[2],t[3],t[4],t[5],t[6], + match &self.jobs[index] {NativeJob::Render(j)=>j.ordinal(),NativeJob::Compute(j)=>j.ordinal()}); + } + if kind==0 && t[0]%512==0 && + crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary) { + let early=match &self.jobs[index] {NativeJob::Render(j)=>j.early_count(),_=>0}; + dev_info!(self.drm.as_ref(),"M3_ORDERED_BATCHES depths={:?} early={} packets={} coalesced_stamps={}\n", self.render_batches,early,self.coalesced_packets,self.coalesced_render_stamps); + let now=::ktime_get(); + dev_info!(self.drm.as_ref(),"M3_GEOMETRY_BATCH ns={} render_count={} overflow={}\n", + now,t[0],self.geometry_overflow); + self.geometry_overflow=0; + for g in &mut self.geometry { + if g.count==0 {continue;} + dev_info!(self.drm.as_ref(),"M3_GEOMETRY ns={} width={} height={} samples={} count={} prepare_ns={} active_ns={} gpu_ns={} tiling_ns={} fragment_ns={} max_fragment_ns={} polling_ns={} retirement_ns={} sleeping_ns={} polls={} stamp_ns={} first_event_ns={} events_ns={} pickup_ns={} tail_ns={} gpu_stamp_ns={} fw_stamp_ns={} queue_done_ns={} queue_read_ns={}\n", + now,g.width,g.height,g.samples,g.count,g.prepare_ns,g.active_ns,g.gpu_ns,g.tiling_ns,g.fragment_ns,g.max_fragment_ns,g.polling_ns,g.retirement_ns,g.sleeping_ns,g.polls,g.stamp_ns,g.first_event_ns,g.events_ns,g.pickup_ns,g.tail_ns,g.completion_ns[0],g.completion_ns[1],g.completion_ns[2],g.completion_ns[3]); + *g=GeometryTiming::default(); + } + } + if self.active.is_empty() { + self.gpu_pending = false; + self.state.health.set_gpu_pending(false); + } + for pair in batch.entries.windows(2) { + if !Arc::ptr_eq(&pair[0].0,&pair[1].0) {self.coalesced_packets+=1;} + } + Ok(()) + } +} +// Keep the large graph on the heap: returning and moving it through probe, +// Registration and Mutex initialization otherwise duplicates it on the bounded +// kernel stack. ManuallyDrop retains the whole allocation on failed ASC stop. +pub(crate) struct Runtime { inner: ManuallyDrop> } +impl Runtime { + #[inline(never)] + pub(crate) fn new(pdev: &platform::Device, device: Device, contents: crate::m3_adt_config::Contents) -> Result { + // On an armed T8122, name a setup failure before the GPU coprocessor is started (RTKit + // state, mailbox transport, the DRM device). Captured as a Copy bool so the closure does + // not borrow `device`, which is moved into Inner below. + let t8122 = crate::t8122_start::is_t8122(device.soc()); + let t6031 = crate::t6031_start::is_t6031(device.soc()); + let coproc_refused = |e: &Error| { + if t8122 { crate::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } + if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), *e); } + }; + device.require_stopped(pdev).inspect_err(coproc_refused)?; + // Reserve before starting ASC. ENOMEM must not drop firmware owners + // while the coprocessor may still access them. Writing the completed + // graph into this allocation is infallible. + let owner = KBox::::new_uninit(GFP_KERNEL)?; + let drm: driver::AsahiDevRef = kernel::drm::Device::new(pdev.as_ref(), driver::AsahiData::new(pdev, None, true)).inspect_err(coproc_refused)?; + let state = m3_rtkit::State::new(pdev, drm.clone(), device.firmware().resources.regions[5]).inspect_err(coproc_refused)?; + let mut transport = rtkit::RtKit::new(pdev.as_ref(), None, 0, state.clone()).inspect_err(coproc_refused)?; + if crate::m3_adt_config::stop_before_asc(pdev.as_ref()) { + if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), ENODEV); } + return Err(ENODEV); + } + // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. + let mut started=false; + let prepared=(|| -> Result<_> { + device.start_asc(pdev)?; + Pin::new(&mut transport).wake()?; + if !state.healthy() || !Pin::new(&transport).is_running() {return Err(EIO);} + for ep in [0x20,0x21] { + if !Pin::new(&mut transport).has_endpoint(ep) {return Err(ENODEV);} + Pin::new(&mut transport).start_endpoint(ep)?; + } + started=true; + // The UAT geometry of the admitted SoC (complete: `Soc::require_complete`). + // SAFETY: the admitted RTKit (J514S on T6030) is awake; no initdata or GPU + // job has been published. Its running PPL handoff owns the peer lock. + let uat=unsafe {mmu::Uat::new_m3_running(&drm,device.soc())}?; + let config=Config::new(&drm,&uat,device.firmware(),&contents)?; + Ok((uat,config)) + })(); + let (uat,config)=match prepared { + Ok(v)=>v, + Err(e)=>{ + crate::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); + if t6031 { crate::t6031_start::prepare_verdict(pdev.as_ref(), started, e); } + state.health.mark_failed(); + if device.stop_asc().is_err() { + unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; + core::mem::forget(transport);core::mem::forget(state); + core::mem::forget(drm);core::mem::forget(device); + } + return Err(e); + } + }; + Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, drm, device, + jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3_pass_layout::SLOTS], + geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false, + boot_step:crate::t8122_start::BootStep::Publish })) }) + } + pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() } + pub(crate) fn health(&self) -> Arc { self.inner.state.health.clone() } + pub(crate) fn new_vm(&mut self, id: u64, range: core::ops::Range) -> Result { + use crate::util::RangeExt; + let reserved_range=0x70_0000_0000..0x80_0000_0000; + if range.overlaps(reserved_range.clone()) {return Err(EINVAL);} + if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } + let vm=self.inner.uat.new_vm(id, range)?; + let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; + vm.install_driver_mappings(KVec::new(),reserved)?; + Ok(vm) + } + pub(crate) fn map_timestamp(&self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { + if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } + bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::agx_memory::TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false) + } + /// Queue a packet and return its fence at once. The scheduler can hand over further + /// ready jobs up to its credit limit while the GPU runs. Only the completion worker + /// publishes: it coalesces render commands of one VM and urgency, and may keep bounded + /// same-VM batches in flight after InitBM retirement. VM/engine changes wait for full + /// retirement. A burst can therefore share a firmware round trip across packets. + pub(crate) fn submit(shared:&crate::m3_drm::Shared,packet:Arc)->Result { + crate::debug::update_debug_flags(); + let mut guard=shared.lock(); + let inner=&mut *Option::as_mut(&mut *guard).ok_or(ENODEV)?.inner; + if !inner.state.healthy() {return Err(EIO);} + if packet.commands.is_empty() || packet.commands.len()>256 { + pr_err!("M3 submit limit: commands={} maximum=256\n",packet.commands.len()); + return Err(E2BIG); + } + // Higher urgency first, FIFO within one urgency, including unfinished packets. + // More urgent work can overtake a continuation between retired batches. + // Packets of one queue share an urgency, so their order is kept. + let urgency=packet.urgency; + let at=policy::pending_position(inner.pending.iter().map(|(p,_)|p.urgency),urgency); + if inner.pending.len()>=policy::MAX_PACKETS {return Err(EBUSY);} + inner.pending.reserve(1,GFP_KERNEL)?; + inner.pending.insert_within_capacity(at,(packet,0)).map_err(|_|ENOMEM)?; + let drm=inner.drm.clone(); + drop(guard); + crate::m3_completion::queue(&drm); + Ok(()) + } + /// Withdraw a packet's unpublished remainder on cancellation or timeout. Its active + /// commands remain owned until retirement; error signaling cannot release a DMA guard. + pub(crate) fn withdraw(shared:&crate::m3_drm::Shared,packet:&Arc)->bool { + let mut guard=shared.lock(); + let Some(runtime)=Option::as_mut(&mut *guard) else {return false;}; + let inner: &mut Inner=&mut runtime.inner; + let removed=inner.pending.iter().position(|(p,_)|Arc::ptr_eq(p,packet)) + .is_some_and(|i|inner.pending.remove(i).is_ok()); + if !inner.packet_active(packet) && !inner.fault_captured {packet.release_guard();} + removed + } + pub(crate) fn completion_wait(&self) -> Arc { self.inner.state.events.clone() } + pub(crate) fn events(&self)->Arc {self.inner.state.events.clone()} + /// One completion-worker step: retire the active batch if it has completed, then publish + /// the next one. The worker drops the runtime lock between steps. + pub(crate) fn service_job(&mut self)->Service { + let chip=self.inner.device.soc().chip_id; + let busy=self.inner.state.healthy() && + (!self.inner.active.is_empty() || !self.inner.pending.is_empty()); + if let Err(error)=self.inner.config.performance_request(chip,busy) { + self.inner.fail_active(error); + return Service::Idle; + } + let result=self.service_job_inner(); + if matches!(result,Service::Idle) || !self.inner.state.healthy() || + (self.inner.active.is_empty() && self.inner.pending.is_empty()) { + let _=self.inner.config.performance_request(chip,false); + } + result + } + fn service_job_inner(&mut self)->Service { + let inner: &mut Inner=&mut *self.inner; + if inner.active.is_empty() { + if inner.fault_captured || !inner.state.healthy() {inner.fail_pending(EIO);return Service::Idle;} + if let Err(error)=inner.config.drain(&inner.drm) { + inner.capture_fault(error);inner.fail_pending(EIO);return Service::Idle; + } + return if inner.publish_next() {Service::Progress} else {Service::Idle}; + } + let messages=inner.state.event_messages.load(Ordering::Acquire); + match inner.poll_active() { + Ok(true)=>{inner.publish_next();Service::Progress}, + Ok(false)=>if inner.publish_next() {Service::Progress} else {Service::Waiting(messages)}, + Err(_)=>{inner.fail_pending(EIO);Service::Idle}, + } + } + pub(crate) fn boot(&mut self, pdev: &platform::Device) -> Result { + let is_t8122 = crate::t8122_start::is_t8122(self.inner.device.soc()); + let is_t6031 = crate::t6031_start::is_t6031(self.inner.device.soc()); + self.inner.boot_step = crate::t8122_start::BootStep::Publish; + let result = self.boot_inner(pdev); + if let Err(error) = result { + // On T8122, log the firmware readiness words on every boot failure, so the verdict's + // "see M3 firmware readiness" always has a line to point at. The ready-wait timeout in + // boot_inner already logs them (on every SoC, as in 12.0), so skip that case here. + let logged = self.inner.boot_step == crate::t8122_start::BootStep::AwaitReady + && error == ETIMEDOUT; + if (is_t8122 || is_t6031) && !logged { + let inner: &mut Inner = &mut *self.inner; + let _ = inner.config.log_ready(&inner.drm); + } + self.inner.capture_fault(error); + } + if is_t8122 { + let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); + crate::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); + } + if is_t6031 { + let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); + crate::t6031_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); + } + result + } + fn boot_inner(&mut self, pdev: &platform::Device) -> Result { + use crate::t8122_start::BootStep; + let root = self.inner.config.root(); + dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root); + self.inner.boot_step = BootStep::Publish; + Pin::new(&mut self.inner.transport).send_message(0x20, 0x0081000000000000 | (root & ((1u64<<44)-1)))?; + self.inner.boot_step = BootStep::DeviceControl; + self.send_control(0x13)?; + self.send_control(9)?; + self.inner.boot_step = BootStep::AwaitReady; + let start = Instant::::now(); + while !self.inner.config.ready()? { + if start.elapsed() >= Delta::from_secs(2) || !self.inner.state.healthy() { + let inner: &mut Inner=&mut *self.inner;let _=inner.config.log_ready(&inner.drm); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_millis(1)); + } + self.inner.boot_step = BootStep::PostReady; + // Import the configured idle policy again after initial power-up. + self.send_control(0x13)?; + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + inner.config.log_ready(&inner.drm)?; + if inner.device.soc().chip_id==0x6030 && + (crate::m3_params::active_power() || crate::m3_params::globals_debug_off()) { + inner.config.log_globals_debug(&inner.drm)?; + } + inner.config.check_pstate(&inner.drm, &inner.device, "after boot")?; + inner.state.health.set_gpu_pending(false); + dev_info!(pdev.as_ref(), "M3: firmware accepted owned initdata and device controls\n"); + Ok(()) + } + fn send_control(&mut self, opcode: u32) -> Result { + let next = match self.inner.config.control(opcode) { + Ok(next) => next, + Err(error) => { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} queue failed ({:?})\n", opcode, error); + return Err(error); + } + }; + let send_start = Instant::::now(); + if let Err(error) = Pin::new(&mut self.inner.transport).send_message(0x21, 0x0083000000000011) { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} mailbox send failed ({:?}), elapsed_ns={}\n", + opcode, error, send_start.elapsed().as_nanos()); + return Err(error); + } + let start = Instant::::now(); + loop { + let inner: &mut Inner = &mut *self.inner; + inner.config.drain(&inner.drm)?; + if !inner.state.healthy() { return Err(EIO); } + if inner.config.control_done(next)? { return Ok(()); } + if start.elapsed() >= Delta::from_secs(2) { + dev_err!(self.inner.drm.as_ref(), "M3: device control {:#x} acknowledgement timed out, elapsed_ns={}\n", + opcode, start.elapsed().as_nanos()); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_millis(1)); + } + } +} +impl Drop for Runtime { + fn drop(&mut self) { + let chip=self.inner.device.soc().chip_id; + let _=self.inner.config.performance_request(chip,false); + self.inner.state.health.mark_failed(); + // Unpublished packets never reach the firmware; a published batch's packets stay + // retained below if the GPU may still use them, but their fences must not hang. + self.inner.fail_pending(ENODEV); + for batch in self.inner.active.iter() {Inner::finish_all(&batch.entries,Err(ENODEV));} + if policy::retain_runtime(self.inner.device.stop_asc().is_err(),self.inner.gpu_pending) { + // SAFETY: a live module reference pins callbacks with retained DMA. + unsafe { kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr()) }; + return; + } + // SAFETY: no GPU command is pending and ASC is stopped. Field order + // drains callbacks before dropping retained commands, UAT and power. + unsafe { ManuallyDrop::drop(&mut self.inner) }; + } +} + +/// Same scaling contract as G16's qualified publication profiler. AP counter +/// rates may differ between SoCs; firmware command timestamps use 24 MHz. +pub(crate) fn physical_counter() -> u64 { + let tick: u64; + let frequency: u64; + // SAFETY: Read architectural counters only; no device state is modified. + unsafe { core::arch::asm!("mrs {f}, cntfrq_el0", "mrs {t}, cntpct_el0", + f = out(reg) frequency, t = out(reg) tick, + options(nomem, nostack, preserves_flags)) }; + let frequency = frequency & 0xffff_ffff; + if frequency == 0 { return 0; } + (tick / frequency) * 24_000_000 + (tick % frequency) * 24_000_000 / frequency +} diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs new file mode 100644 index 00000000000000..9bd08d0257b0c9 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Bounded M3 compute and graphics submissions through the common DRM scheduler/syncobjs. +use kernel::{c_str,dma_fence::*,drm::sched,prelude::*,sync::{Arc,Mutex},new_mutex,xarray}; +use crate::{file,agx_uapi,mmu,queue,m3_drm::Shared}; +use core::sync::atomic::{AtomicBool, Ordering}; +#[derive(Default)] +pub(crate) struct Completion; +#[vtable] +impl FenceOps for Completion { + fn get_driver_name<'a>(self:&'a FenceObject)->&'a CStr {c_str!("asahi")} + fn get_timeline_name<'a>(self:&'a FenceObject)->&'a CStr {c_str!("m3-native")} +} +pub(crate) struct Destination { mapping: Arc, offset: usize } +impl Destination { + fn resolve(objects: Pin<&xarray::XArray>>, + timestamp: agx_uapi::UapiTimestamp) -> Result> { + if timestamp.handle == 0 { return Ok(None); } + let guard = objects.lock(); + let object = guard.get(timestamp.handle.try_into()?).ok_or(ENOENT)?; + match &*object { + file::Object::TimestampBuffer(mapping) => { + let offset = timestamp.offset as usize; + if offset & 7 != 0 || offset.checked_add(8).ok_or(EOVERFLOW)? > mapping.size() { + return Err(EINVAL); + } + Ok(Some(Self { mapping: mapping.clone(), offset })) + } + } + } + fn firmware_address(&self) -> u64 { self.mapping.iova() + self.offset as u64 } +} + +#[derive(Copy,Clone)] +pub(crate) enum Command { + Compute(crate::agx_compute::Control), + Render{command:agx_uapi::UapiRenderCommand,usc:u64}, +} +pub(crate) struct Packet { + pub(crate) vm:mmu::Vm, + pub(crate) commands:KVec, + timestamps:KVec<[Option;4]>, + pub(crate) attachments:KVec, + pub(crate) wide_visibility:KVec, + completion:UserFence, + finish_claimed: AtomicBool, + /// Queue urgency (3 = UAPI realtime .. 0 = UAPI low): more urgent packets are published first. + pub(crate) urgency:u8, + /// Who submitted the packet, for the failure log only. + pub(crate) owner:Owner, + vm_job:Pin>>>, +} +/// The process and thread that submitted a packet, recorded in the submit ioctl. Only logged: +/// a renamed task leaves a stale name, which is harmless. +#[derive(Copy,Clone)] +pub(crate) struct Owner {pub(crate) pid:i32,process:[u8;16],thread:[u8;16]} +impl Owner { + fn current()->Self { + let task=kernel::current!(); + let leader=task.group_leader(); + Self {pid:leader.pid(),process:crate::m3_client::task_comm(leader), + thread:crate::m3_client::task_comm(task)} + } + fn name(comm:&[u8;16])->&str { + let len=comm.iter().position(|&b|b==0).unwrap_or(comm.len()); + core::str::from_utf8(&comm[..len]).unwrap_or("?") + } + pub(crate) fn process(&self)->&str {Self::name(&self.process)} + pub(crate) fn thread(&self)->&str {Self::name(&self.thread)} +} +impl Packet { + pub(crate) fn profile_id(&self)->usize {self as *const Self as usize} + pub(crate) fn timestamp_addresses(&self,index:usize)->[u64;2] { + core::array::from_fn(|i|self.timestamps[index][i].as_ref().map_or(0,Destination::firmware_address)) + } + pub(crate) fn render_timestamp_addresses(&self,index:usize)->[[u64;2];2] { + core::array::from_fn(|stage| core::array::from_fn(|i| + self.timestamps[index][stage*2+i].as_ref().map_or(0,Destination::firmware_address))) + } + /// Called only after all published commands retired or before any publication. + pub(crate) fn release_guard(&self) {drop(self.vm_job.lock().take());} + pub(crate) fn retired(&self) {self.release_guard();self.finish(Ok(()));} + pub(crate) fn finish(&self,result:Result) { + // Retirement, cancellation and a queued scheduler timeout can race. + // Claim error/signal ownership once; this flag is NOT proof that the + // fence is signaled or that hardware DMA has stopped. + if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } + if let Err(e)=result {if let Ok(status) = self.vm.agx_status() { status.record(e.to_errno()); }self.completion.set_error(e);} + // A scheduler job/fence and the runtime's last packet can outlive + // hardware retirement. They must not keep the VM's active-job count + // nonzero: GEM closes otherwise accumulate across completed frames. + // On failure the runtime quarantines this guard with the packet until + // firmware has provably stopped; signalling an error is not retirement. + crate::agx_memory::publish(); + let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); + let before=if profile {crate::m3_runtime::physical_counter()} else {0}; + self.completion.signal(); + if profile { + let after=crate::m3_runtime::physical_counter(); + let status=match result {Ok(())=>0,Err(e)=>e.to_errno()}; + pr_info!("M3_PACKET_FENCE vm={} packet={:#x} commands={} signal_before_tick={} signal_after_tick={} status={} counter_hz=24000000\n", + self.vm.id(),self.profile_id(),self.commands.len(),before,after,status); + } + } +} +pub(crate) struct Job {shared:Shared,packet:Arc} +/// Timeout state of a [`Job`], owned by the scheduler's timeout worker. +pub(crate) struct JobTimeout {shared:Shared,packet:Arc} +impl sched::JobImpl for Job { + type TimeoutData = JobTimeout; + fn timeout_data(&self)->JobTimeout { + JobTimeout {shared:self.shared.clone(),packet:self.packet.clone()} + } + fn run(job:&mut Self)->Result> { + // Queue the packet and return its fence; the completion worker publishes and retires + // it, so the scheduler can hand over further ready jobs meanwhile. + if let Err(e)=crate::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { + pr_err!("M3 scheduler: submission failed {:?}\n",e); + // submit() failed before this packet could be published. + job.packet.release_guard();job.packet.finish(Err(e)); + } + Ok(Some(Fence::from_fence(&job.packet.completion))) + } + fn timed_out(job:&mut JobTimeout,_finished:bool)->sched::Status { + // Retain the publishing branch's explicit diagnostic override. + if !crate::m3_params::timeout_nohang() { + if let Some(r)=Option::as_mut(&mut *job.shared.lock()) {r.health().mark_failed();} + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(ETIMEDOUT));return sched::Status::NoDevice; + } + let guard=job.shared.lock(); + let Some(runtime)=guard.as_ref() else { + job.packet.finish(Err(ENODEV)); + return sched::Status::NoDevice; + }; + // The scheduler's deadline covers the whole packet, including time queued behind + // other batches. The completion worker owns each batch's bounded hardware-completion + // checks. Reinsert a healthy packet instead of turning a late timeout into a global + // device loss or corrupting an already signaled fence. + if runtime.health().healthy() { return sched::Status::NoHang; } + drop(guard); + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(EIO)); + sched::Status::NoDevice + } + fn cancel(job:&mut Self) { + crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + job.packet.finish(Err(ECANCELED)); + } +} +struct AddressSpace<'a>(&'a mmu::Vm); +impl agx_uapi::GpuAddressSpace for AddressSpace<'_> { + fn covers(&self,address:u64,size:u64,access:agx_uapi::GpuAccess)->bool { + let Some(end)=address.checked_add(size) else {return false;}; + let (r,w)=match access {agx_uapi::GpuAccess::Read=>(true,false), + agx_uapi::GpuAccess::Write=>(false,true),agx_uapi::GpuAccess::ReadWrite=>(true,true)}; + address>=0x4000 && end<=(1u64<<42)-0x8000 + && !self.0.driver_range_overlaps(address..end) && self.0.covers_range(address,size,r,w) + } +} +pub(crate) struct Queue { + entity:sched::Entity,_scheduler:Arc>, + shared:Shared,vm:mmu::Vm,usc:u64,fences:FenceContexts, + /// Whether the SoC advertises early tiling (`m3_soc::Features::fragment_dependency`). + early_tiling:bool, + urgency:u8, +} +impl Queue { + pub(crate) fn new(shared:Shared,scheduler:Arc>,vm:mmu::Vm,priority:u32,usc:u64, + early_tiling:bool)->Result { + agx_uapi::QueueUscWindow{base:usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}.validate().map_err(|_|EINVAL)?; + // file.rs passes REALTIME - UAPI priority: 3 is UAPI LOW, 0 is UAPI REALTIME. + let urgency=3u8.saturating_sub(priority.min(3) as u8); + let priority=match priority {3=>sched::Priority::Low,2=>sched::Priority::Normal, + 1=>sched::Priority::High,0=>sched::Priority::Kernel,_=>return Err(EINVAL)}; + Ok(Self {entity:sched::Entity::new(&scheduler,priority)?,_scheduler:scheduler, + shared,vm,usc,fences:FenceContexts::new(1,c_str!("asahi_m3_queue"),kernel::static_lock_class!())?, + early_tiling,urgency}) + } +} +impl queue::Queue for Queue { + fn submit(&mut self,_id:u64,mut syncs:KVec,in_sync_count:usize, + raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { + let vm_job=self.vm.retain_m3_job()?; + let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); + while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { + if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { + if let Err(e)=validate_render(payload,self.usc,&AddressSpace(&self.vm),self.early_tiling) { + pr_err!("M3 render rejected {:?}: flags={:#x} size={}x{} layers={} samples={} utile={}x{} sample_size={} depth={:#x} stencil={:#x} zls={:#x} samplers={} heap={:#x}\n",e,payload.flags,payload.width,payload.height,payload.layers,payload.samples,payload.utile_width,payload.utile_height,payload.sample_size,payload.depth.base,payload.stencil.base,payload.zls_control,payload.sampler_count,payload.sampler_heap); + return Err(e); + } + agx_uapi::validate_attachments(&AddressSpace(&self.vm),&vertex_attachments).map_err(|_|EINVAL)?; + agx_uapi::validate_attachments(&AddressSpace(&self.vm),&fragment_attachments).map_err(|_|EINVAL)?; + commands.push(Command::Render{command:payload,usc:self.usc},GFP_KERNEL)?; + timestamps.push([Destination::resolve(objects,payload.vertex_timestamps.start)?, + Destination::resolve(objects,payload.vertex_timestamps.end)?, + Destination::resolve(objects,payload.fragment_timestamps.start)?, + Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; + flushes.push(crate::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + visibility.push(false,GFP_KERNEL)?; + continue; + } + let agx_uapi::ParsedHardwareCommand::Compute{mut payload,attachments,..}=command else {return Err(ENOTSUPP);}; + // Other engines and helpers need their own M3 firmware translation. + let wide=kernel::uapi::drm_asahi_compute_flags_DRM_ASAHI_COMPUTE_WIDE_VISIBILITY as u32; + if payload.flags & !wide !=0 || payload.sampler_count!=0 || payload.sampler_heap!=0 { + pr_err!("M3 compute rejected: flags={:#x} samplers={} heap={:#x} attachments={} timestamps={}/{}\n",payload.flags,payload.sampler_count,payload.sampler_heap,attachments.as_slice().len(),payload.timestamps.start.handle,payload.timestamps.end.handle); + return Err(ENOTSUPP); + } + visibility.push(payload.flags & wide !=0,GFP_KERNEL)?; + // The common translator validates the original command/attachment + // addresses; this M3 scheduling hint changes none of them. + payload.flags &= !wide; + flushes.push(encode_attachments(&self.vm,&attachments)?,GFP_KERNEL)?; + let c=agx_uapi::translate_compute_command(payload,attachments, + agx_uapi::QueueUscWindow{base:self.usc,user_start:0x4000,user_end:(1u64<<42)-0x8000}, + &AddressSpace(&self.vm)).map_err(|e| { + pr_err!("M3 compute translation rejected {:?}: stream={:#x}..{:#x} helper={:?}\n",e,payload.control_stream_base,payload.control_stream_end,payload.helper); + EINVAL + })?; + crate::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; + timestamps.push([Destination::resolve(objects,payload.timestamps.start)?, + Destination::resolve(objects,payload.timestamps.end)?,None,None],GFP_KERNEL)?; + commands.push(Command::Compute(crate::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; + } + parser.finish().map_err(|_|EINVAL)?; + let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, + completion:self.fences.new_fence(0,Completion)?.into(), + finish_claimed:AtomicBool::new(false),urgency:self.urgency,owner:Owner::current(), + vm_job:KBox::pin_init(new_mutex!(Some(vm_job)),GFP_KERNEL)?},GFP_KERNEL)?; + let mut job=self.entity.new_job(1,Job{shared:self.shared.clone(),packet})?; + for sync in syncs.drain(0..in_sync_count) {if let Some(f)=sync.fence {job.add_dependency(f)?;}} + let mut job=job.arm();let finished=job.fences().finished();job.push(); + for mut sync in syncs {if let Some(chain)=sync.chain_fence.take() { + sync.syncobj.add_point(chain,&finished,sync.timeline_value); + } else {sync.syncobj.replace_fence(Some(&finished));}} + Ok(()) + } +} + +fn encode_attachments(vm: &mmu::Vm, list: &agx_uapi::UapiAttachmentList) + -> Result { + use crate::agx_attachments::{Attachments, CAPACITY}; + use agx_uapi::{GpuAddressSpace, GpuAccess}; + if list.as_slice().len()>CAPACITY {return Err(E2BIG);} + let mut ranges = [(0, 0); CAPACITY]; + for (range, entry) in ranges.iter_mut().zip(list.as_slice()) { + *range = Attachments::range(entry.address, entry.size).map_err(|_| EINVAL)?; + if !AddressSpace(vm).covers(range.0, range.1, GpuAccess::Write) { + return Err(EINVAL); + } + } + let mut out=Attachments::new(&ranges[..list.as_slice().len()]).map_err(|_| EINVAL)?; + // G15S RTKit-2419 uses the legacy 0x17 cache attribute; retain the + // M4 range normalization and ownership checks, not its 0x18 attribute. + for i in 0..list.as_slice().len() {out.0[i*16+12..i*16+14].copy_from_slice(&0x17u16.to_le_bytes());} + Ok(out) +} + +fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpace<'_>, + early_tiling: bool) -> Result { + use agx_uapi::{GpuAddressSpace, GpuAccess}; + if r.sample_size % 8 != 0 {return Err(ENOTSUPP);} + let empty = agx_uapi::UapiHelperProgram { binary:0, config:0, data:0 }; + // Early tiling (render flag bit 5) only where it is advertised. + let stage_scope=if early_tiling && *crate::module_parameters::m3_early_tiling.value()!=0 {1<<5} else {0}; + if r.flags & !(2|16|(1<<18)|stage_scope) != 0 || !matches!(r.samples, 1 | 2 | 4) + || r.sampler_count != 0 || r.sampler_heap != 0 + || r.vertex_helper != empty || r.fragment_helper != empty + || r.stencil.compression_base != 0 || r.stencil.compression_stride != 0 + || r.ppp_control & !0x203 != 0 { return Err(ENOTSUPP); } + // Native 26A428 descriptors witness the ZLS metadata registers and mirrors + // for single-sample, single-layer depth. Keep other layouts unqualified. + if (r.depth.compression_base != 0 || r.stencil.compression_base != 0) + && (r.samples != 1 || r.layers != 1) {return Err(ENOTSUPP);} + // sample_size describes color tilebuffer storage. Depth/stencil-only + // passes legitimately require zero bytes of color storage. + if r.layers == 0 || r.layers > 2048 || r.width == 0 || r.height == 0 || r.width > 16384 || r.height > 16384 + || !matches!(r.utile_width,16|32) || !matches!(r.utile_height,16|32) + || u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)>32768 { + return Err(EINVAL); + } + crate::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; + if r.vdm_base & 3 != 0 || !space.covers(r.vdm_base,4,GpuAccess::Read) + || r.scissor_base == 0 || r.scissor_base & 7 != 0 || !space.covers(r.scissor_base,8,GpuAccess::Read) { + return Err(EINVAL); + } + for (p,access) in [(r.depth_bias_base,GpuAccess::Read),(r.occlusion_query_base,GpuAccess::Write)] { + if p != 0 && (p & 7 != 0 || !space.covers(p,8,access)) { return Err(EINVAL); } + } + for zls in [r.depth,r.stencil] { agx_uapi::validate_zls(space,zls,r.layers).map_err(|_| EINVAL)?; } + let window = agx_uapi::QueueUscWindow { base:usc,user_start:0x4000,user_end:((1u64 << 42) - 2 * mmu::UAT_PGSZ as u64) }; + for p in [r.background,r.end_of_tile,r.partial_background,r.partial_end_of_tile] { + window.full_program(space,p.usc).map_err(|_| EINVAL)?; + if !space.covers(usc+u64::from(p.usc & !63),64,GpuAccess::Read) { return Err(EINVAL); } + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_thermal.rs b/drivers/gpu/drm/asahi/m3/m3_thermal.rs new file mode 100644 index 00000000000000..59a72187c96990 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/m3_thermal.rs @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Thermal limit of the M3 runtime backend's GPU performance cap (`asahi.m3_thermal`). +//! +//! Without it (`off`, the default with the built-in InitData image) the firmware is given a +//! fixed highest performance state (5 of 8 with device-tree InitData: 1056 of 1380 MHz on +//! J516S) and nothing watches temperatures. The firmware's own die temperature controller is +//! off in the InitData this driver builds. With device-tree InitData the default is `on`; +//! `asahi.m3_thermal=off` restores the fixed cap exactly. +//! +//! With it, the InitData gives the firmware the whole table as its highest state (the +//! "ceiling"), and what bounds the firmware is the runtime cap: the Globals performance-state +//! cap and power-interface state targets, which the firmware keeps reading while it runs (a +//! host moves the cap with a single store, with no message). The runtime cap starts at the +//! fixed cap of `off` (the "safe cap") and never goes below it, so at worst the GPU runs as it +//! does without the limit. +//! +//! - `hold`: the runtime cap stays at the safe cap for the whole boot. The GPU then runs exactly +//! as with `off`, provided the firmware follows the runtime cap; the per-job check below +//! proves or disproves that before `on` is used. +//! - `on`: the runtime cap follows the hottest SoC die temperature, read by a work item every +//! 500 ms while the GPU is busy (every 2 s when idle) from the thermal zones in [`ZONES`] +//! that exist: the SMC's die sensors and the PMP's CPU cluster sensors (see [`ZONES`]): +//! * at or above `critical` (`hot` + 8 C): the safe cap at once; +//! * at or above `hot` (`asahi.m3_thermal_hot`, default 80 C): one state lower, at most every +//! 500 ms and once per reading; +//! * at or below `cool` (`hot` - 10 C): one state higher, at most every 2 s and once per +//! reading, up to the ceiling; +//! * no zone, a zone that cannot be read, a reading outside 10..=125 C, one older than 3 s +//! or one that has not changed at all for 60 s: the safe cap at once. +//! +//! The defaults leave a wide margin below the limits the boot firmware's device tree gives this +//! machine (a 94 C GPU die target and a 111 C SoC-hot temperature), because the sensors may +//! read cooler than the GPU's hottest spot (the PMP's are CPU cluster sensors), and this limit +//! reacts in steps of hundreds of milliseconds while the firmware changes state within the cap +//! much faster. +//! +//! The cap is changed only by the job thread, after a job has retired, right before the state +//! the firmware reports is checked. After a lowering the previous cap is tolerated for 1 s. A +//! reported state above the runtime cap in 20 consecutive checks spanning at least 3 s means +//! the firmware does not follow the runtime cap, so this limit cannot work: the GPU is then +//! marked failed, as for a state above the ceiling. +//! +//! The decisions are in `m3_thermal_policy`, which host tests drive; this module reads the +//! sensors, publishes the cap and logs. + +use core::sync::atomic::{AtomicBool, Ordering}; + +use kernel::{ + bindings, c_str, device, + error::to_result, + new_spinlock, + prelude::*, + str::CStr, + sync::{Arc, SpinLock}, + time::{msecs_to_jiffies, Instant, Monotonic}, + workqueue::{self, impl_has_delayed_work, new_delayed_work, DelayedWork, WorkItem}, +}; + +use crate::{ + m3_adt_config::PstatePolicy, + m3_params::{self, ThermalMode}, + m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, +}; + +/// The thermal zones read, each the hottest of a set of SoC die sensors: the SMC's die sensors +/// (registered by the SMC hwmon driver) and the PMP's CPU cluster sensors (registered by the +/// PMP report driver once the PMP runs). Zones that do not exist are skipped; a zone that +/// exists but cannot be read fails the reading. The M3 Pro boards (J514S and J516S) need +/// both zones: on a J516S the SMC die sensors alone stayed below 67 C while the PMP hotspot +/// exceeded 85 C and the cap stepped down. Without the PMP running, the cap stays safe. +/// Every T8122 board requires the SMC die zone; T8122 has no PMP hotspot zone. +const ZONES: [&CStr; 2] = [c_str!("macsmc_soc_die"), c_str!("apple_pmp_hotspot")]; +// Do not raise its cap on a partial reconstruction of that sensor set. +const J516_REQUIRED_ZONES: u32 = 3; +fn required_zones() -> u32 { + let Some(root) = kernel::of::root() else { return J516_REQUIRED_ZONES; }; + let Ok(compatible) = root.get_property::>(c_str!("compatible")) else { + return J516_REQUIRED_ZONES; + }; + crate::m3_thermal_policy::required_zones(&compatible) +} + +/// Sensor polling period while the GPU is busy and while it is idle, in ms. +const POLL_BUSY_MS: u32 = 500; +const POLL_IDLE_MS: u32 = 2000; +/// The GPU counts as busy for this long after a job has retired, in ns. +const BUSY_WINDOW_NS: i64 = 5 * S; +/// Lines logging cap changes per boot, and the period of the summary line while busy. +const LOG_LINES: u32 = 256; +const SUMMARY_EVERY_NS: i64 = 30 * S; + +type Time = Instant; + +/// Nanoseconds from `base` to `t`. +fn ns(base: Time, t: Time) -> i64 { + (t - base).as_nanos() +} + +struct SensorState { + reading: Reading, + /// When the GPU last retired a job, in ns from the sensor's base. + busy: i64, +} + +/// Reads [`ZONES`] from a self-rearming delayed work item, so the job thread never waits for the +/// SMC. The work item holds a reference, so it stays valid after the governor is dropped; it +/// then stops at its next run. +#[pin_data] +struct Sensor { + required_zones: u32, + /// The origin of the sensor's and the governor's times. + base: Time, + stopped: AtomicBool, + #[pin] + state: SpinLock, + #[pin] + work: DelayedWork, +} + +impl_has_delayed_work! { + impl HasDelayedWork for Sensor { self.work } +} + +/// One reading of the zone `name`, in millidegrees Celsius; ENODEV if there is no such zone. +fn read_zone(name: &CStr) -> Result { + let mut temp: i32 = 0; + // SAFETY: name is NUL-terminated and temp is a valid output pointer. The + // thermal core acquires a device reference under its registry lock and + // keeps it through sampling, so zone unregistration cannot free it here. + to_result(unsafe { bindings::thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; + Ok(temp) +} + +/// The hottest reading of the zones in [`ZONES`] that exist, and which ones were read. +fn read_zones(required: u32) -> Result<(i32, u32)> { + let mut hottest: Option = None; + let mut zones = 0; + for (i, name) in ZONES.iter().enumerate() { + match read_zone(name) { + Ok(temp) => { + hottest = Some(hottest.map_or(temp, |h| h.max(temp))); + zones |= 1 << i; + } + Err(e) if e == ENODEV => {} + Err(e) => return Err(e), + } + } + if zones & required != required { + return Err(ENODEV); + } + hottest.map(|t| (t, zones)).ok_or(ENODEV) +} + +impl WorkItem for Sensor { + type Pointer = Arc; + + fn run(this: Arc) { + if this.stopped.load(Ordering::Acquire) { + return; + } + let result = read_zones(this.required_zones); + let at = ns(this.base, Time::now()); + let reading = match result { + Ok((mc, zones)) => Reading::Temp { mc, at, zones }, + Err(e) => Reading::Failed { + errno: e.to_errno(), + at, + }, + }; + let busy = { + let mut state = this.state.lock(); + state.reading = reading; + at - state.busy < BUSY_WINDOW_NS + }; + let delay = if busy { POLL_BUSY_MS } else { POLL_IDLE_MS }; + let _ = workqueue::system().enqueue_delayed(this, msecs_to_jiffies(delay)); + } +} + +fn describe(source: Source) -> &'static str { + match source { + Source::Unknown | Source::NoReading => "no reading yet", + Source::Ok(_) => "reading", + Source::Failed(e) if e == ENODEV.to_errno() => "no SoC die temperature zone", + Source::Failed(_) => "a SoC die temperature zone cannot be read", + Source::Stale => "the reading is older than 3 s", + Source::Implausible => "the reading is outside 10..=125 C", + Source::Stuck => "the reading has not changed for 60 s", + } +} + +fn describe_why(why: Why) -> &'static str { + match why { + Why::Critical => "critical", + Why::Hot => "hot", + Why::Cool => "cool", + Why::NoTemperature(source) => describe(source), + } +} + +/// Degrees Celsius with one decimal, for the log. +struct Celsius(i32); + +impl kernel::fmt::Display for Celsius { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + let sign = if self.0 < 0 { "-" } else { "" }; + let v = self.0.unsigned_abs(); + write!(f, "{}{}.{} C", sign, v / 1000, (v % 1000) / 100) + } +} + +/// The runtime cap and the thermal limit that moves it. +pub(crate) struct Governor { + mode: ThermalMode, + /// Without the limit (`off`): the cap, which never changes. + fixed: u32, + policy: Policy, + freqs: [u32; 16], + base: Time, + sensor: Option>, + source: Source, + last_summary: Option, + /// The highest state the firmware reported since the last summary. + peak: u32, + lines: u32, +} + +impl Governor { + /// The governor for `policy`. Starts the sensor work item unless the limit is off. + pub(crate) fn new(dev: &device::Device, pstates: &PstatePolicy) -> Result { + let (hot_c, clamped) = m3_params::thermal_hot_param(); + let base = Time::now(); + let mut governor = Governor { + mode: pstates.thermal, + fixed: pstates.max, + policy: Policy::new( + pstates.thermal == ThermalMode::On, + pstates.safe, + pstates.max, + hot_c as i32 * 1000, + ), + freqs: pstates.freqs, + base, + sensor: None, + source: Source::Unknown, + last_summary: None, + peak: 0, + lines: 0, + }; + if governor.mode == ThermalMode::Off { + return Ok(governor); + } + if clamped { + dev_info!( + dev, + "M3: asahi.m3_thermal_hot clamped to {} C ({}..={} C)\n", + hot_c, + m3_params::THERMAL_HOT_MIN, + m3_params::THERMAL_HOT_MAX + ); + } + let sensor = Arc::pin_init( + pin_init!(Sensor { + required_zones: required_zones(), + base, + stopped: AtomicBool::new(false), + state <- new_spinlock!(SensorState { + reading: Reading::None, + busy: 0, + }), + work <- new_delayed_work!("asahi::M3ThermalSensor"), + }), + GFP_KERNEL, + )?; + let _ = workqueue::system().enqueue_delayed(sensor.clone(), msecs_to_jiffies(POLL_BUSY_MS)); + governor.sensor = Some(sensor); + let p = &governor.policy; + let (cool, hot, critical) = p.thresholds(); + match governor.mode { + ThermalMode::Hold => dev_info!( + dev, + "M3: thermal limit holding (asahi.m3_thermal=hold): the firmware may use states 1..={} ({} MHz) and the runtime cap stays at {} ({} MHz) for this boot; the SoC die temperature is logged only\n", + p.ceiling(), + governor.mhz(p.ceiling()), + p.safe(), + governor.mhz(p.safe()) + ), + _ => dev_info!( + dev, + "M3: thermal limit on (asahi.m3_thermal=on): runtime cap {}..={} ({}..={} MHz) from the hottest SoC die temperature of zones macsmc_soc_die and apple_pmp_hotspot: up while at or below {}, down at or above {}, cap {} at or above {} or without a fresh reading\n", + p.safe(), + p.ceiling(), + governor.mhz(p.safe()), + governor.mhz(p.ceiling()), + Celsius(cool), + Celsius(hot), + p.safe(), + Celsius(critical) + ), + } + Ok(governor) + } + + fn mhz(&self, state: u32) -> u32 { + self.freqs.get(state as usize).copied().unwrap_or(0) + } + + fn cap(&self) -> u32 { + match self.mode { + ThermalMode::Off => self.fixed, + ThermalMode::Hold | ThermalMode::On => self.policy.cap(), + } + } + + fn note_source(&mut self, dev: &device::Device, source: Source, temp: Option) { + if source == self.source { + return; + } + self.source = source; + let safe = self.policy.safe(); + match (source, temp) { + (Source::Ok(zones), Some(t)) => { + let smc = if zones & 1 != 0 { + " macsmc_soc_die (SMC die sensors)" + } else { + "" + }; + let pmp = if zones & 2 != 0 { + " apple_pmp_hotspot (PMP CPU cluster sensors)" + } else { + "" + }; + dev_info!( + dev, + "M3: thermal: SoC die temperature {} from zones:{}{}\n", + Celsius(t), + smc, + pmp + ) + } + (Source::Failed(e), _) => dev_info!( + dev, + "M3: thermal: {} (error {}); runtime cap at most {} ({} MHz)\n", + describe(source), + e, + safe, + self.mhz(safe) + ), + _ => dev_info!( + dev, + "M3: thermal: {}; runtime cap at most {} ({} MHz)\n", + describe(source), + safe, + self.mhz(safe) + ), + } + } + + fn summary(&mut self, dev: &device::Device, now: i64, temp: Option) { + if self + .last_summary + .is_some_and(|t| now - t < SUMMARY_EVERY_NS) + { + return; + } + self.last_summary = Some(now); + let peak = core::mem::take(&mut self.peak); + let cap = self.policy.cap(); + let (raised, lowered) = self.policy.counts(); + let (safe, ceiling) = (self.policy.safe(), self.policy.ceiling()); + match temp { + Some(t) => dev_info!( + dev, + "M3: thermal: SoC die {}, runtime cap {} ({} MHz) of {}..={}, firmware peak state {} since the last summary, raised {} and lowered {} times so far\n", + Celsius(t), + cap, + self.mhz(cap), + safe, + ceiling, + peak, + raised, + lowered + ), + None => dev_info!( + dev, + "M3: thermal: no SoC die temperature ({}), runtime cap {} ({} MHz) of {}..={}, firmware peak state {} since the last summary, raised {} and lowered {} times so far\n", + describe(self.source), + cap, + self.mhz(cap), + safe, + ceiling, + peak, + raised, + lowered + ), + } + } + + fn log_change(&mut self, dev: &device::Device, change: Change, temp: Option) { + if self.lines < LOG_LINES { + match temp { + Some(t) => dev_info!( + dev, + "M3: thermal: runtime cap {} -> {} ({} MHz): SoC die {} ({})\n", + change.from, + change.to, + self.mhz(change.to), + Celsius(t), + describe_why(change.why) + ), + None => dev_info!( + dev, + "M3: thermal: runtime cap {} -> {} ({} MHz): {}\n", + change.from, + change.to, + self.mhz(change.to), + describe_why(change.why) + ), + } + } else if self.lines == LOG_LINES { + dev_info!( + dev, + "M3: thermal: further runtime cap changes are not logged\n" + ); + } + self.lines = self.lines.saturating_add(1); + } + + /// Recompute the runtime cap after a job. Returns the new cap if it changed; the caller + /// publishes it. + pub(crate) fn update(&mut self, dev: &device::Device, now: Time) -> Option { + let Some(sensor) = &self.sensor else { + return None; + }; + let reading = sensor.state.lock().reading; + let now = ns(self.base, now); + let (usable, change) = self.policy.update(now, reading); + let temp = usable.ok().map(|(t, _, _)| t); + match usable { + Err(source) => self.note_source(dev, source, None), + Ok((t, _, zones)) => self.note_source(dev, Source::Ok(zones), Some(t)), + } + self.summary(dev, now, temp); + let change = change?; + self.log_change(dev, change, temp); + Some(change.to) + } + + /// The highest state the firmware may report now: the runtime cap, or the highest cap of + /// the grace period after a lowering. + pub(crate) fn allowed(&self, now: Time) -> u32 { + match self.mode { + ThermalMode::Off => self.fixed, + ThermalMode::Hold | ThermalMode::On => self.policy.allowed(ns(self.base, now)), + } + } + + /// Account one check of the reported state; `over` says whether it stayed above + /// [`Self::allowed`], `peak` is the highest state it reported. Fails with ERANGE when the + /// firmware does not follow the runtime cap. + pub(crate) fn note_check( + &mut self, + dev: &device::Device, + now: Time, + over: bool, + peak: u32, + ) -> Result { + self.peak = self.peak.max(peak); + if let Some((count, span)) = self.policy.note_check(ns(self.base, now), over) { + let cap = self.cap(); + dev_err!( + dev, + "M3: thermal: the firmware reported a state above the runtime cap {} ({} MHz) in {} consecutive checks over {} ms: it does not follow the runtime cap; marking the GPU failed\n", + cap, + self.mhz(cap), + count, + span / 1_000_000 + ); + return Err(ERANGE); + } + Ok(()) + } + + /// Note that the GPU retired a job, so the sensor is read at the busy rate. + pub(crate) fn busy(&self, now: Time) { + if let Some(sensor) = &self.sensor { + sensor.state.lock().busy = ns(self.base, now); + } + } +} + +impl Drop for Governor { + fn drop(&mut self) { + if let Some(sensor) = &self.sensor { + sensor.stopped.store(true, Ordering::Release); + } + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs new file mode 100644 index 00000000000000..f9c3ff08006d5b --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The parameters of the T8122 start experiment (`t8122_start`): their text, defaults, accepted +//! values and their meaning. Plain data only, so that the host tests run it (tools/asahi). + +/// A parameter that is not given. +pub(crate) const UNSET: u64 = u64::MAX; +/// A parameter whose text is not one of its accepted values. The experiment refuses to start +/// with it rather than fall back to a default. +pub(crate) const INVALID: u64 = u64::MAX - 1; + +/// The Fender window sizes: the ADT sgx reg[1] size less 0x28000 (the difference on T6030), and +/// the whole ADT sgx reg[1]. +pub(crate) const FENDER_RULE: u32 = 0x10_4000; +pub(crate) const FENDER_ADT: u32 = 0x12_c000; +/// The GPU clock-generator offsets in SGX: the T8122 table's (`T8122_IO_MAPPINGS`, the default), +/// and the T6030 one. +pub(crate) const CLOCK_GEN_E1C: u64 = 0xe1_c000; +pub(crate) const CLOCK_GEN_E5C: u64 = 0xe5_c000; +/// The SGX setup write of T6030 and T8122 (offset, value). +pub(crate) const T6030_SGX_SETUP: (usize, u32) = (0xd1_4000, 0x7_0001); +/// The default unit masks: the T8122 table's (`m3_soc::T8122_HWDATA_B`), which are T6030's. +pub(crate) const UNIT_MASK_A: u64 = 0x7_0000_0003; +pub(crate) const UNIT_MASK_B: u32 = 0x7; +/// No unit-mask bit outside these is accepted: the default masks. +pub(crate) const UNIT_MASK_A_LIMIT: u64 = 0x7_0000_0003; +pub(crate) const UNIT_MASK_B_LIMIT: u64 = 0x7; +/// The default performance-state ceiling: the lowest two states. +pub(crate) const PSTATE_CAP: u32 = 2; +/// The highest performance-state ceiling the experiment accepts. The same as the driver's own +/// limit when the fast-die temperature controller is off (`m3_adt_config::ADT_MAX_PSTATE_LIMIT`): +/// J613/J615 have no SoC die temperature zone yet, so there is no thermal feedback above it. +pub(crate) const PSTATE_CAP_MAX: u32 = 5; + +/// Parse a decimal or `0x`-prefixed hexadecimal `u64`. A leading sign is not accepted. +fn number(text: &str) -> Option { + if text.starts_with(['+', '-']) { + return None; + } + match text.strip_prefix("0x").or_else(|| text.strip_prefix("0X")) { + Some(hex) if !hex.starts_with(['+', '-']) => u64::from_str_radix(hex, 16).ok(), + Some(_) => None, + None => text.parse::().ok(), + } +} + +/// Parse a parameter's text: one of `names`, else a number. Anything else, and the two +/// reserved values, give [`INVALID`]. +fn parse(text: &str, names: &[(&str, u64)]) -> u64 { + let text = text.trim(); + if let Some(&(_, value)) = names.iter().find(|(name, _)| *name == text) { + return value; + } + number(text).filter(|v| *v < INVALID).unwrap_or(INVALID) +} + +pub(crate) fn parse_number(text: &str) -> u64 { + parse(text, &[]) +} + +pub(crate) fn parse_fender(text: &str) -> u64 { + parse(text, &[("rule", FENDER_RULE as u64), ("adt", FENDER_ADT as u64)]) +} + +pub(crate) fn parse_clkgen(text: &str) -> u64 { + parse(text, &[("e1c", CLOCK_GEN_E1C), ("e5c", CLOCK_GEN_E5C), ("none", 0)]) +} + +pub(crate) fn parse_sgx_setup(text: &str) -> u64 { + parse(text, &[("none", 0), ("t6030", 1)]) +} + +/// The parameters as given: [`UNSET`], [`INVALID`] or a value. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Raw { + pub(crate) start: u64, + pub(crate) initdata_version: u64, + pub(crate) fender: u64, + pub(crate) clkgen: u64, + pub(crate) sgx_setup: u64, + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u64, + pub(crate) pstate_cap: u64, +} + +impl Raw { + /// No parameter given. + #[cfg(test)] + pub(crate) const NONE: Raw = Raw { + start: UNSET, + initdata_version: UNSET, + fender: UNSET, + clkgen: UNSET, + sgx_setup: UNSET, + unit_mask_a: UNSET, + unit_mask_b: UNSET, + pstate_cap: UNSET, + }; + + /// Whether any parameter other than `asahi.t8122_start` was given. + pub(crate) fn values_given(&self) -> bool { + [ + self.initdata_version, + self.fender, + self.clkgen, + self.sgx_setup, + self.unit_mask_a, + self.unit_mask_b, + self.pstate_cap, + ] + .iter() + .any(|v| *v != UNSET) + } +} + +/// What `asahi.t8122_start` asks for. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Start { + /// Not given, or 0. + Off, + /// 1. + On, + /// Anything else. + Invalid, +} + +pub(crate) fn start(raw: &Raw) -> Start { + match raw.start { + UNSET | 0 => Start::Off, + 1 => Start::On, + _ => Start::Invalid, + } +} + +/// The GPU clock-generator IO mapping. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum ClockGen { + /// SGX + the offset, mapped read-only for the firmware. + At(u64), + /// No mapping: HwDataB slot 29 stays zero. + Absent, +} + +/// The values of one boot. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Values { + pub(crate) initdata_version: u64, + pub(crate) fender: u32, + pub(crate) clock_gen: ClockGen, + pub(crate) sgx_setup: Option<(usize, u32)>, + pub(crate) unit_mask_a: u64, + pub(crate) unit_mask_b: u32, + pub(crate) pstate_cap: u32, +} + +/// A parameter given with a value it does not accept: its name and what it accepts. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Refusal { + pub(crate) name: &'static str, + pub(crate) accepts: &'static str, +} + +/// A parameter's value, its default when not given; None when invalid. +fn given(param: u64, default: u64) -> Option { + match param { + UNSET => Some(default), + INVALID => None, + v => Some(v), + } +} + +/// Resolve the parameters. The first parameter with a value it does not accept is the refusal. +pub(crate) fn resolve(raw: &Raw) -> Result { + let refuse = |name, accepts| Refusal { name, accepts }; + let initdata_version = given(raw.initdata_version, crate::m3_firmware::G15_V14_8_3_INITDATA) + .filter(|v| *v != 0) + .ok_or(refuse("t8122_initdata_version", "a nonzero number, at most 0xfffffffffffffffd"))?; + let fender = match given(raw.fender, FENDER_RULE as u64) { + Some(v) if v == FENDER_RULE as u64 || v == FENDER_ADT as u64 => v as u32, + _ => return Err(refuse("t8122_fender", "0x104000 or rule, 0x12c000 or adt")), + }; + let clock_gen = match given(raw.clkgen, CLOCK_GEN_E1C) { + Some(0) => ClockGen::Absent, + Some(v) if v == CLOCK_GEN_E1C || v == CLOCK_GEN_E5C => ClockGen::At(v), + _ => return Err(refuse("t8122_clkgen", "e1c, e5c or none")), + }; + let sgx_setup = match given(raw.sgx_setup, 1) { + Some(0) => None, + Some(1) => Some(T6030_SGX_SETUP), + _ => return Err(refuse("t8122_sgx_setup", "none or t6030")), + }; + let unit_mask_a = given(raw.unit_mask_a, UNIT_MASK_A) + .filter(|v| *v != 0 && *v & !UNIT_MASK_A_LIMIT == 0) + .ok_or(refuse("t8122_unit_mask_a", "nonzero, no bits outside 0x700000003"))?; + let unit_mask_b = given(raw.unit_mask_b, UNIT_MASK_B as u64) + .filter(|v| *v != 0 && *v & !UNIT_MASK_B_LIMIT == 0) + .ok_or(refuse("t8122_unit_mask_b", "nonzero, no bits outside 0x7"))? as u32; + let pstate_cap = given(raw.pstate_cap, PSTATE_CAP as u64) + .filter(|v| (1..=u64::from(PSTATE_CAP_MAX)).contains(v)) + .ok_or(refuse("t8122_pstate_cap", "1 to 5"))? as u32; + Ok(Values { + initdata_version, + fender, + clock_gen, + sgx_setup, + unit_mask_a, + unit_mask_b, + pstate_cap, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn with(edit: impl Fn(&mut Raw)) -> Raw { + let mut raw = Raw { start: 1, ..Raw::NONE }; + edit(&mut raw); + raw + } + + #[test] + fn nothing_given_is_off_and_gives_no_values() { + assert_eq!(start(&Raw::NONE), Start::Off); + assert!(!Raw::NONE.values_given()); + assert_eq!(start(&with(|r| r.start = 0)), Start::Off); + assert_eq!(start(&with(|_| {})), Start::On); + assert_eq!(start(&with(|r| r.start = 2)), Start::Invalid); + assert_eq!(start(&with(|r| r.start = INVALID)), Start::Invalid); + assert!(!with(|_| {}).values_given()); + assert!(with(|r| r.pstate_cap = 3).values_given()); + assert!(with(|r| r.fender = INVALID).values_given()); + } + + #[test] + fn defaults_are_the_plan_baseline() { + let v = resolve(&with(|_| {})).unwrap(); + assert_eq!(v.initdata_version, 0x0c08_e21e_8380_0490); + assert_eq!(v.fender, 0x10_4000); + assert_eq!(v.clock_gen, ClockGen::At(0xe1_c000)); + assert_eq!(v.sgx_setup, Some((0xd1_4000, 0x7_0001))); + assert_eq!((v.unit_mask_a, v.unit_mask_b), (0x7_0000_0003, 7)); + assert_eq!(v.pstate_cap, 2); + } + + #[test] + fn every_alternative_resolves() { + let v = resolve(&with(|r| { + r.initdata_version = parse_number("0x123456789abcdef0"); + r.fender = parse_fender("adt"); + r.clkgen = parse_clkgen("e5c"); + r.sgx_setup = parse_sgx_setup("none"); + r.unit_mask_a = parse_number("1"); + r.unit_mask_b = parse_number("0x1"); + r.pstate_cap = parse_number("5"); + })) + .unwrap(); + assert_eq!(v.initdata_version, 0x1234_5678_9abc_def0); + assert_eq!(v.fender, 0x12_c000); + assert_eq!(v.clock_gen, ClockGen::At(0xe5_c000)); + assert_eq!(v.sgx_setup, None); + assert_eq!((v.unit_mask_a, v.unit_mask_b, v.pstate_cap), (1, 1, 5)); + let v = resolve(&with(|r| r.clkgen = parse_clkgen("none"))).unwrap(); + assert_eq!(v.clock_gen, ClockGen::Absent); + // Numbers and names give the same values. + assert_eq!(parse_fender("0x104000"), parse_fender("rule")); + assert_eq!(parse_fender(" 0x12C000 "), parse_fender("adt")); + assert_eq!(parse_clkgen("0xe5c000"), parse_clkgen("e5c")); + assert_eq!(parse_clkgen("0xe1c000"), parse_clkgen("e1c")); + assert_eq!(parse_clkgen("0"), parse_clkgen("none")); + assert_eq!(parse_sgx_setup("1"), parse_sgx_setup("t6030")); + } + + #[test] + fn unaccepted_values_refuse_and_name_the_parameter() { + let cases: [(fn(&mut Raw), &str); 15] = [ + (|r| r.initdata_version = parse_number("g15s"), "t8122_initdata_version"), + (|r| r.initdata_version = 0, "t8122_initdata_version"), + (|r| r.fender = parse_fender("0x144000"), "t8122_fender"), + (|r| r.fender = parse_fender("big"), "t8122_fender"), + (|r| r.clkgen = parse_clkgen("0xe00000"), "t8122_clkgen"), + (|r| r.clkgen = parse_clkgen("e5"), "t8122_clkgen"), + (|r| r.sgx_setup = parse_sgx_setup("2"), "t8122_sgx_setup"), + (|r| r.unit_mask_a = parse_number("0x700000007"), "t8122_unit_mask_a"), + (|r| r.unit_mask_a = 0, "t8122_unit_mask_a"), + (|r| r.unit_mask_b = parse_number("0xf"), "t8122_unit_mask_b"), + (|r| r.unit_mask_b = 0, "t8122_unit_mask_b"), + (|r| r.pstate_cap = 0, "t8122_pstate_cap"), + (|r| r.pstate_cap = 6, "t8122_pstate_cap"), + (|r| r.pstate_cap = 16, "t8122_pstate_cap"), + (|r| r.pstate_cap = parse_number("-1"), "t8122_pstate_cap"), + ]; + for (edit, name) in cases { + assert_eq!(resolve(&with(edit)).unwrap_err().name, name); + } + } + + #[test] + fn pstate_cap_accepts_one_to_five_only() { + for cap in 1..=PSTATE_CAP_MAX { + assert_eq!(resolve(&with(|r| r.pstate_cap = u64::from(cap))).unwrap().pstate_cap, cap); + } + for cap in [0u64, 6, 8, 15, 16] { + assert!(resolve(&with(|r| r.pstate_cap = cap)).is_err(), "cap {cap}"); + } + assert_eq!(PSTATE_CAP_MAX, 5); + } + + #[test] + fn a_leading_sign_is_rejected() { + for text in ["+1", "-1", "0x+1", "0x-1", "+0x10"] { + assert_eq!(parse_number(text), INVALID, "{text:?}"); + } + assert!(resolve(&with(|r| r.pstate_cap = parse_number("+3"))).is_err()); + } + + #[test] + fn reserved_numbers_and_garbage_parse_as_invalid() { + for text in ["", "0x", "x1", "18446744073709551615", "18446744073709551614", "0x1g"] { + assert_eq!(parse_number(text), INVALID, "{text:?}"); + } + assert_eq!(parse_number("18446744073709551613"), INVALID - 1); + assert_eq!(parse_number(" 0X10 "), 16); + } + + #[test] + fn default_masks_are_within_the_limits() { + assert_eq!(UNIT_MASK_A & !UNIT_MASK_A_LIMIT, 0); + assert_eq!(u64::from(UNIT_MASK_B) & !UNIT_MASK_B_LIMIT, 0); + assert_eq!(UNIT_MASK_A_LIMIT, 0x7_0000_0003); + assert_eq!(UNIT_MASK_B_LIMIT, 0x7); + } +} diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs new file mode 100644 index 00000000000000..cf0ee658e8da50 --- /dev/null +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The T8122 (M3, G15G) start, `asahi.t8122_start=1`. +//! +//! The T8122 table (`m3_soc::T8122`) is complete, but the runtime has not passed on an M3 +//! MacBook Air yet, so the SoC is refused unless `asahi.t8122_start=1` arms the start. On a T8122 +//! whose GPU the boot loader handed over (resource admission passed), an armed start uses the +//! table's values, with a performance-state cap. Each value can be overridden for a lab A/B by +//! rebooting with a parameter: +//! +//! | Parameter | Default | Others | +//! |---|---|---| +//! | `asahi.t8122_initdata_version` | `0x0c08e21e83800490`, the G15 14.8.3 version | any | +//! | `asahi.t8122_fender` | `0x104000` (`rule`) | `0x12c000` (`adt`) | +//! | `asahi.t8122_clkgen` | `e1c`: SGX+0xe1c000, read-only | `e5c`: SGX+0xe5c000, read-only; `none` | +//! | `asahi.t8122_sgx_setup` | `t6030`: SGX+0xd14000 = 0x70001 | `none` | +//! | `asahi.t8122_unit_mask_a` | `0x700000003` | nonzero, within `0x700000003` | +//! | `asahi.t8122_unit_mask_b` | `0x7` | nonzero, within `0x7` | +//! | `asahi.t8122_pstate_cap` | `2` | 1 to 5 | +//! +//! The defaults are the T8122 table's values. A parameter given with a value it does not accept +//! refuses the start. Without `asahi.t8122_start=1` nothing here runs, and T8122 is refused. +//! Nothing here applies to any other SoC. + +use core::mem::offset_of; + +use kernel::{device, prelude::*}; + +use crate::{ + fw::initdata::raw, + initdata::G15RuntimeHwDataB, + m3_adt_config::T8122_IO_MAPPINGS, + m3_init_storage::{self as storage, IoMap}, + m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, + pgtable::{prot, Prot}, + t8122_knobs::{ + self as knobs, ClockGen, Start, Values, CLOCK_GEN_E1C, CLOCK_GEN_E5C, FENDER_ADT, + FENDER_RULE, + }, +}; + +/// The GPU register window (SGX) of T8122. +const SGX: u64 = 0x2_9000_0000; +/// Index of the Fender window and of the GPU clock generator in [`T8122_IO_MAPPINGS`]. +const FENDER: usize = 0; +const CLOCK_GEN: usize = 11; +/// The HwDataB slot of the GPU clock generator. +const CLOCK_GEN_SLOT: usize = 29; + + +/// The firmware IO mappings with the Fender window `fender` and the clock generator at +/// SGX + `clock_gen`. +const fn mappings(fender: u32, clock_gen: u64) -> [IoMapping; 12] { + let mut m = T8122_IO_MAPPINGS; + m[FENDER].2 = fender; + m[FENDER].3 = fender; + m[CLOCK_GEN].1 = SGX + clock_gen; + m +} + +/// The runtime IO maps, laid out once for the larger Fender window so that no slot's firmware +/// VA depends on the parameters. +const LAYOUT: [IoMap; 12] = + storage::pack_iomaps(&mappings(FENDER_ADT, CLOCK_GEN_E1C), storage::IOMAP_BASE); + +const fn iomaps(fender: u32, clock_gen: u64) -> [IoMap; 12] { + let mut m = LAYOUT; + m[FENDER].size = fender as usize; + m[CLOCK_GEN].physical = SGX + clock_gen; + m +} + +/// Every Fender window and clock-generator address, in [`Experiment::variant`] order. +static IO_MAPPINGS: [[IoMapping; 12]; 4] = [ + mappings(FENDER_RULE, CLOCK_GEN_E5C), + mappings(FENDER_RULE, CLOCK_GEN_E1C), + mappings(FENDER_ADT, CLOCK_GEN_E5C), + mappings(FENDER_ADT, CLOCK_GEN_E1C), +]; +static IOMAPS: [[IoMap; 12]; 4] = [ + iomaps(FENDER_RULE, CLOCK_GEN_E5C), + iomaps(FENDER_RULE, CLOCK_GEN_E1C), + iomaps(FENDER_ADT, CLOCK_GEN_E5C), + iomaps(FENDER_ADT, CLOCK_GEN_E1C), +]; + +/// Whether each IO map holds its IO mapping: same slot, the mapping's 16 KiB-aligned physical +/// base, its subpage offset, and room for its total size. +const fn maps_cover(mappings: &[IoMapping; 12], maps: &[IoMap; 12]) -> bool { + let mut i = 0; + while i < 12 { + let (slot, phys, total, _, _) = mappings[i]; + let io = maps[i]; + if io.slot != slot + || io.physical + io.offset as u64 != phys + || io.offset + total as usize > io.size + || io.size % 0x4000 != 0 + { + return false; + } + i += 1; + } + true +} + +const _: () = { + // The default variant's IO mappings are the T8122 table's. + let defaults = mappings(FENDER_RULE, CLOCK_GEN_E1C); + let mut j = 0; + while j < 12 { + let (a, b) = (defaults[j], T8122_IO_MAPPINGS[j]); + assert!(a.0 == b.0 && a.1 == b.1 && a.2 == b.2 && a.3 == b.3 && a.4 == b.4); + j += 1; + } + // The T8122 table: the Fender window at the rule size, the clock generator last, at + // SGX+0xe1c000, 16 KiB, read-only. + assert!(T8122_IO_MAPPINGS[FENDER].0 == 0 && T8122_IO_MAPPINGS[FENDER].1 == SGX + 0xd0_0000); + assert!(T8122_IO_MAPPINGS[FENDER].2 == FENDER_RULE); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].0 == CLOCK_GEN_SLOT); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].1 == SGX + CLOCK_GEN_E1C); + assert!(T8122_IO_MAPPINGS[CLOCK_GEN].2 == 0x4000 && !T8122_IO_MAPPINGS[CLOCK_GEN].4); + assert!(CLOCK_GEN + 1 == T8122_IO_MAPPINGS.len()); + // Both windows are whole pages and hold the Fender scratch area (+0x60000..+0x80000). + assert!(FENDER_RULE % 0x4000 == 0 && FENDER_ADT % 0x4000 == 0 && FENDER_RULE >= 0x8_0000); + assert!(FENDER_RULE < FENDER_ADT); + // Both clock-generator blocks are whole pages inside the first 16 MiB of SGX. + assert!(CLOCK_GEN_E5C % 0x4000 == 0 && CLOCK_GEN_E1C % 0x4000 == 0); + assert!(CLOCK_GEN_E5C + 0x4000 <= 0x100_0000 && CLOCK_GEN_E1C + 0x4000 <= 0x100_0000); + let mut i = 0; + while i < 4 { + assert!(maps_cover(&IO_MAPPINGS[i], &IOMAPS[i])); + i += 1; + } + // The default unit masks are the T8122 table's, and the limits allow them. + assert!(T8122_HWDATA_B.unit_mask_a == knobs::UNIT_MASK_A); + assert!(T8122_HWDATA_B.unit_mask_b == knobs::UNIT_MASK_B); + assert!(knobs::UNIT_MASK_A & !knobs::UNIT_MASK_A_LIMIT == 0); + assert!(knobs::UNIT_MASK_B as u64 & !knobs::UNIT_MASK_B_LIMIT == 0); + assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_a) == 0x17c0); + assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_b) == 0x17c8); + // An armed start skips `require_complete`. The only `unported` item of the T8122 table is the + // opt-in itself; if another is added, this fails the build until the start accounts for it. + assert!(crate::m3_soc::T8122.unported.len() == 1); +}; + +/// The values of one armed boot. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Experiment(Values); + +/// Whether `soc` is the T8122 table. +pub(crate) fn is_t8122(soc: &Soc) -> bool { + core::ptr::eq(soc, &crate::m3_soc::T8122) +} + +impl Experiment { + /// Index of this boot's IO-mapping table in [`IO_MAPPINGS`] and [`IOMAPS`]. + fn variant(&self) -> usize { + let adt = usize::from(self.0.fender == FENDER_ADT); + let e1c = usize::from(self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C)); + 2 * adt + e1c + } + + /// How many IO mappings this boot gives the firmware: all, or all but the clock generator. + fn mapping_count(&self) -> usize { + match self.0.clock_gen { + ClockGen::At(_) => CLOCK_GEN + 1, + ClockGen::Absent => CLOCK_GEN, + } + } + + /// The InitData version given to the firmware (`asahi.t8122_initdata_version`). + pub(crate) fn initdata_version(&self) -> u64 { + self.0.initdata_version + } + + /// The firmware IO mappings of the InitData. + pub(crate) fn io_mappings(&self) -> &'static [IoMapping] { + &IO_MAPPINGS[self.variant()][..self.mapping_count()] + } + + /// The runtime's mapping of each of them. With the default Fender window and clock generator + /// they are the T8122 table's own IO maps. + pub(crate) fn iomaps(&self) -> &'static [IoMap] { + if self.0.fender == FENDER_RULE && self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C) { + return &crate::m3_adt_config::T8122_IOMAPS; + } + &IOMAPS[self.variant()][..self.mapping_count()] + } + + /// The register block the IO mapping of `slot` must lie in, given the HwConfig's block + /// `base` for it: the selected clock generator for slot 29. + pub(crate) fn io_block(&self, slot: usize, base: u64) -> u64 { + match self.0.clock_gen { + ClockGen::At(offset) if slot == CLOCK_GEN_SLOT => SGX + offset, + _ => base, + } + } + + /// The HwDataB slots the firmware maps read-only, as a bit mask: the clock generator's. + pub(crate) fn read_only_slots(&self) -> u32 { + match self.0.clock_gen { + ClockGen::At(_) => 1 << CLOCK_GEN_SLOT, + ClockGen::Absent => 0, + } + } + + /// The runtime HwDataB words: the T8122 table's. The unit masks are written over them + /// ([`Self::unit_masks`]). + pub(crate) fn hwdata_b(&self) -> &'static G15RuntimeHwDataB { + &T8122_HWDATA_B + } + + /// The HwDataB unit masks A (+0x17c0) and B (+0x17c8). + pub(crate) fn unit_masks(&self) -> (u64, u32) { + (self.0.unit_mask_a, self.0.unit_mask_b) + } + + /// The SGX write (offset, value) made before the firmware starts, if any. + pub(crate) fn sgx_setup(&self) -> Option<(usize, u32)> { + self.0.sgx_setup + } + + /// The highest performance state the firmware may use (`asahi.t8122_pstate_cap`). + pub(crate) fn pstate_cap(&self) -> u32 { + self.0.pstate_cap + } +} + +/// Firmware page protection of the IO map of `slot`, with the read-only slots `read_only`. +pub(crate) fn mmio_prot(read_only: u32, slot: usize) -> Prot { + if slot < 32 && read_only & (1 << slot) != 0 { + prot::PROT_FW_MMIO_RO + } else { + prot::PROT_FW_MMIO_RW + } +} + +/// Decide, after resource admission, whether `soc` starts with the experiment values. +/// +/// Any SoC but T8122: Ok(None), silently. T8122 without `asahi.t8122_start=1`: Ok(None), so the +/// caller refuses it as before (one extra line when experiment parameters were given). T8122 +/// with `asahi.t8122_start=1`: the experiment, after logging every value, or a refusal for a +/// parameter value it does not accept. +pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> { + if !is_t8122(soc) { + return Ok(None); + } + let raw = crate::m3_params::t8122_params(); + match knobs::start(&raw) { + Start::On => {} + Start::Off => { + if raw.values_given() { + dev_info!( + dev, + "M3 G15G start: asahi.t8122_* values given without asahi.t8122_start=1; ignored\n" + ); + } + return Ok(None); + } + Start::Invalid => { + dev_err!( + dev, + "M3 G15G start: refused: asahi.t8122_start has a value it does not accept (0 or 1); GPU startup disabled\n" + ); + return Err(ENODEV); + } + } + // Admission has required the boot loader's handoff: its firmware segments and reserved + // regions, and on this SoC its power configuration. + if !soc.power_from_boot_loader || soc.firmware.is_none() || soc.hwcfg.is_none() { + dev_err!(dev, "M3 G15G start: refused: the T8122 table is not the one this experiment expects\n"); + return Err(ENODEV); + } + let v = knobs::resolve(&raw).map_err(|refusal| { + dev_err!( + dev, + "M3 G15G start: refused: asahi.{} has a value it does not accept ({}); GPU startup disabled\n", + refusal.name, + refusal.accepts + ); + ENODEV + })?; + let e = Experiment(v); + dev_warn!( + dev, + "M3 G15G start: armed (asahi.t8122_start=1, boot loader handoff admitted): initdata_version={:#018x} fender={:#x} clkgen={} sgx_setup={} unit_mask_a={:#x} unit_mask_b={:#x} pstate_cap={}\n", + v.initdata_version, + v.fender, + clkgen_label(v.clock_gen, v.fender), + match v.sgx_setup { + Some(_) => "t6030(sgx+0xd14000=0x70001)", + None => "none", + }, + v.unit_mask_a, + v.unit_mask_b, + v.pstate_cap + ); + dev_warn!( + dev, + "M3 G15G start: the T8122 table's InitData words, HwData placement and register lists; {} IO mappings at firmware VA {:#x}\n", + e.mapping_count(), + storage::IOMAP_BASE + ); + // J613/J615 have no SoC die temperature zone yet, so a cap above the no-feedback limit runs + // the fanless GPU without any temperature feedback (`m3_thermal` holds the cap but gets no + // reading). The runtime cap and the firmware ceiling stay equal, because the cap is at most + // the no-feedback limit. + if v.pstate_cap > knobs::PSTATE_CAP { + dev_warn!( + dev, + "M3 G15G start: pstate_cap={} is above {}: no SoC die temperature zone on this board, so states up to {} run with no temperature feedback\n", + v.pstate_cap, + knobs::PSTATE_CAP, + v.pstate_cap + ); + } + Ok(Some(e)) +} + +/// The armed line's clock-generator field. It says `ro` for the firmware's own slot-29 mapping, +/// and adds that the Fender window reaches the block read-write when the chosen window covers it +/// (SGX+0xe1c000 lies under the 0x12c000 window but not the 0x104000 one; SGX+0xe5c000 under +/// neither). +fn clkgen_label(clock_gen: ClockGen, fender: u32) -> &'static str { + let fender_end = 0xd0_0000u64 + u64::from(fender); + let reaches = |off: u64| off + 0x4000 <= fender_end; + match clock_gen { + ClockGen::At(CLOCK_GEN_E1C) if reaches(CLOCK_GEN_E1C) => { + "e1c(sgx+0xe1c000,ro; also rw in the fender window)" + } + ClockGen::At(CLOCK_GEN_E1C) => "e1c(sgx+0xe1c000,ro)", + ClockGen::At(_) if reaches(CLOCK_GEN_E5C) => { + "e5c(sgx+0xe5c000,ro; also rw in the fender window)" + } + ClockGen::At(_) => "e5c(sgx+0xe5c000,ro)", + ClockGen::Absent if reaches(CLOCK_GEN_E1C) => { + "none (but the fender window reaches sgx+0xe1c000 rw)" + } + ClockGen::Absent => "none", + } +} + +/// Log, in an armed start, that the probe stopped at `stage` before the firmware started. +pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, stage: &str, error: Error) { + if experiment.is_some() { + dev_err!( + dev, + "M3 G15G start: refused at {} ({:?}); the firmware was not started, see the lines above\n", + stage, + error + ); + } +} + +/// Log, on a T8122 armed with `asahi.t8122_start=1`, that resource admission refused it. +pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) { + if is_t8122(soc) && knobs::start(&crate::m3_params::t8122_params()) == Start::On { + dev_err!( + dev, + "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n" + ); + } +} + +// One `M3 G15G verdict:` line per outcome of an armed start, so that a tester can tell them +// apart from the log alone. They run on T8122 only (callers check `is_t8122`); nothing here +// changes what the runtime does. + +/// The GPU coprocessor's start or the driver's InitData upload failed (`m3_runtime::Runtime::new`). +/// `started`: the coprocessor ran and offered its endpoints. +pub(crate) fn prepare_verdict(dev: &device::Device, soc: &Soc, started: bool, error: Error) { + if !is_t8122(soc) { + return; + } + if started { + dev_err!( + dev, + "M3 G15G verdict: driver-refused ({:?}): the GPU firmware is up, but the driver did not publish the InitData (UAT, upload checks or thermal setup, see above)\n", + error + ); + } else { + dev_err!( + dev, + "M3 G15G verdict: firmware-boot-failed ({:?}): the GPU coprocessor did not start, or its RTKit did not offer endpoints 0x20/0x21; no InitData was published\n", + error + ); + } +} + +/// The step the firmware boot sequence reached (`m3_runtime::Runtime::boot_inner`). The verdict +/// uses it to name a failure as a publish, a device-control or an InitData-acknowledgement +/// problem, rather than lumping them all under "initdata-rejected". +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum BootStep { + /// About to send, or sending, the InitData publish message (endpoint 0x20). + Publish, + /// A device-control message (0x13 or 9) before the firmware signalled ready. + DeviceControl, + /// Waiting (up to 2 s) for the firmware to write its ready words. + AwaitReady, + /// The firmware signalled ready; a control or check after that. + PostReady, +} + +/// The firmware's answer to the published InitData (`m3_runtime::Runtime::boot`). `accepted`: +/// the firmware wrote its ready words. `step`: the furthest step the boot reached. +pub(crate) fn boot_verdict( + dev: &device::Device, + version: u64, + step: BootStep, + accepted: bool, + crashed: bool, + result: Result, +) { + let Err(e) = result else { + dev_info!( + dev, + "M3 G15G verdict: firmware-running: the firmware accepted the InitData (version {:#x}) and the device controls; no job has run yet\n", + version + ); + return; + }; + // The firmware signalled ready, then a later control or check failed: the InitData was + // accepted, so this is not a rejection. + if accepted || step == BootStep::PostReady { + if crashed { + dev_err!( + dev, + "M3 G15G verdict: firmware-running-check-failed ({:?}): the firmware accepted the InitData (version {:#x}) then crashed during a post-boot control or check; see the crash lines above\n", + e, + version + ); + } else { + dev_err!( + dev, + "M3 G15G verdict: firmware-running-check-failed ({:?}): the firmware accepted the InitData (version {:#x}), then a post-boot control or check failed (see above)\n", + e, + version + ); + } + return; + } + match step { + BootStep::Publish => dev_err!( + dev, + "M3 G15G verdict: publish-failed ({:?}): the InitData publish message (endpoint 0x20) could not be sent; the firmware was not told about the InitData (version {:#x})\n", + e, + version + ), + BootStep::DeviceControl => dev_err!( + dev, + "M3 G15G verdict: device-control-failed ({:?}): a device-control queue, mailbox send or acknowledgement failed after the InitData (version {:#x}) was published; see the device-control diagnostic above\n", + e, + version + ), + // AwaitReady (and PostReady is handled above). + _ if crashed => dev_err!( + dev, + "M3 G15G verdict: initdata-rejected ({:?}): the firmware crashed after the InitData (version {:#x}) was published; see the crash lines above\n", + e, + version + ), + _ => dev_err!( + dev, + "M3 G15G verdict: initdata-rejected ({:?}): the firmware did not acknowledge the InitData (version {:#x}) within 2 s (its ready words stayed unset); see M3 firmware readiness above\n", + e, + version + ), + } +} + +/// A GPU coprocessor setup step failed before the firmware could be started +/// (`m3_runtime::Runtime::new`): the RTKit state, the mailbox transport or the DRM device. +pub(crate) fn coproc_setup_refused(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G start: refused at the GPU coprocessor setup (RTKit/mailbox) ({:?}); the firmware was not started\n", + error + ); +} + +/// The firmware ran a job above the performance-state cap (`m3_runtime`, the after-job pstate +/// check). The `M3: firmware ... stays above the cap ...` line precedes this. +pub(crate) fn cap_violated_verdict(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G verdict: cap-violated ({:?}): after a job the firmware reported a performance state above the cap; the GPU was marked failed (see the 'stays above the cap' line above)\n", + error + ); +} + +/// A job-path step failed outside the completion wait (`m3_runtime::Runtime::execute`): a submit, +/// a doorbell, or a post-completion read. No dispatch outcome was determined. +pub(crate) fn job_setup_failed_verdict(dev: &device::Device, error: Error) { + dev_err!( + dev, + "M3 G15G verdict: job-failed-before-wait ({:?}): a submit, doorbell or diagnostic step failed, so no dispatch outcome was determined; see the lines above\n", + error + ); +} + +/// A job that failed (`m3_runtime::Inner::fail`). `primary` is ETIMEDOUT when it did not retire +/// within the runtime's per-batch bound; `start_stamp` is the job's first GPU start (CDM +/// dispatch) timestamp, if readable; `pstate` is the GPU performance-state register, if readable. +pub(crate) fn job_failed_verdict( + dev: &device::Device, + primary: Error, + start_stamp: Option, + pstate: Option, + crashed: bool, +) { + if primary != ETIMEDOUT || crashed { + dev_err!( + dev, + "M3 G15G verdict: job-faulted ({:?}): the firmware crashed, reported an error, or the GPU reported a fault while the job ran; see the lines around this one\n", + primary + ); + return; + } + // A nonzero start timestamp means the firmware dispatched the job on the GPU, so this was not + // a dispatch gap however the power state reads now (the GPU may have idled off since). The + // stamp and pstate print as `Some(hex)` or `None`. + if start_stamp.is_some_and(|start| start != 0) { + dev_err!( + dev, + "M3 G15G verdict: job-ran-completion-missed: the job was dispatched (GPU start timestamp {:x?}) but its completion was not seen within the per-batch bound (pstate register {:x?})\n", + start_stamp, + pstate + ); + return; + } + // The GPU start timestamp is 0 (or unreadable): the job was not dispatched. + match pstate { + Some(p) if p & 0xf == 0 => dev_err!( + dev, + "M3 G15G verdict: job-accepted-never-dispatched: the job was not dispatched (GPU start timestamp {:x?}) and the GPU reads powered down (pstate register {:#x})\n", + start_stamp, + p + ), + Some(p) => dev_err!( + dev, + "M3 G15G verdict: job-timed-out-powered: the job did not retire with the GPU powered (pstate register {:#x}, GPU start timestamp {:x?}); see the engine snapshot below\n", + p, + start_stamp + ), + None => dev_err!( + dev, + "M3 G15G verdict: job-timed-out: the job did not retire (pstate register and GPU start timestamp both unreadable)\n" + ), + } +} + +/// The first job of a kind (0 render, 1 compute) retired (`m3_runtime::Runtime::execute`): +/// `span` is its first start and last end GPU timestamp (24 MHz ticks), if readable. +pub(crate) fn job_completed_verdict(dev: &device::Device, kind: usize, gpu_ns: u64, span: Option<[u64; 2]>) { + let what = if kind == 0 { "render" } else { "compute" }; + match span { + Some([start, end]) if start != 0 && end != 0 => dev_info!( + dev, + "M3 G15G verdict: job-completed: the first {} job finished, GPU timestamps {:#x}..{:#x}, {} ns of GPU time\n", + what, + start, + end, + gpu_ns + ), + Some([start, end]) => dev_err!( + dev, + "M3 G15G verdict: job-retired-without-timestamps: the first {} job retired, but its GPU timestamps are {:#x}..{:#x}\n", + what, + start, + end + ), + None => dev_err!( + dev, + "M3 G15G verdict: job-retired-without-timestamps: the first {} job retired, but its GPU timestamps are unreadable\n", + what + ), + } +} From ad52c9b317f605911ee6f40cdfda7b6435623d6c Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 01:11:35 -0400 Subject: [PATCH 2/8] drm/asahi: m3: start the T8122 runtime on 7.2 Wire the restored 7.1.12 M3 runtime to the 7.2 driver behind CONFIG_DRM_ASAHI_M3_T8122 (default n), so the M3 MacBook Air (J613, J615) GPU runs again on its 14.8.3 firmware. A T8122 probe still passes the G15G identity gate and the backend selection first, and the runtime still starts only with asahi.t8122_start=1; otherwise the probe is refused as before. Other GPUs are not affected: T6030/T6031 keep the G15 backend or the refusal, and the T6030/T6031 start paths and the T6031 start experiment are dropped from the runtime. On 7.2 the runtime: - registers through drm::Registration as a gpu::Gpu backend, like G15 and G17, and is stopped on unbind while its device resources are live; the completion worker moves from the DRM device data into the runtime's RTKit state; - keeps user mappings alive across accepted work with the shared VM job lifetime (VmJobGuard, retain_job, deferred unmaps), the generalized form of 7.1.12's M3 VM job lifetime, on VMs created with tracked status; - adopts the firmware's UAT as the G15 backend does on 7.2 (Uat::new_adopted and g15::uat: the PPL exchange, TTBAT context 0, the host window at root entry 2 through the shared L2) instead of 7.1.12's M3-only handoff mode in mmu.rs/pgtable.rs. The firmware- walked pages are cleaned to the point of coherency, the host window is linked before the InitData and each batch reach the firmware, and context 0 is invalidated when a job's command aliases go. The option therefore depends on DRM_ASAHI_G15; - refuses VMs to userspace that did not ask GET_PARAMS for the whole global parameter structure (7.1.12's per-file client gate), through a new gpu::Gpu::admit_vm hook that admits every client elsewhere. Smaller 7.2 adaptations: the IoMem/Devres, IoSysMap and RTKit buffer APIs; Queue::submit's registration argument; quiet MMIO mappings in an mmu::m3 helper module; the M3 debug flags, F32::to_bits, Completion::complete/try_wait_for_completion/wait_for_completion_timeout and RtKit::is_running/is_crashed, which 7.2 dropped; the M3 module parameters move into m3_params, and m3_soc.rs gets back its full 7.1.12 table, cfg-gated with the runtime. asahi.m3_expose=0 is refused before anything starts. The devcoredump fault snapshot is dropped (7.2's kernel crate has no devcoredump binding): the first failure is logged. Generated-by: LLM Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/Kconfig | 15 + drivers/gpu/drm/asahi/debug.rs | 7 + drivers/gpu/drm/asahi/driver.rs | 57 ++++ drivers/gpu/drm/asahi/file.rs | 14 + drivers/gpu/drm/asahi/float.rs | 5 + drivers/gpu/drm/asahi/g15/mod.rs | 4 +- drivers/gpu/drm/asahi/g15/regs.rs | 15 + drivers/gpu/drm/asahi/g15/uat.rs | 33 +- drivers/gpu/drm/asahi/gpu.rs | 6 + drivers/gpu/drm/asahi/m3/agx_fault.rs | 87 ----- drivers/gpu/drm/asahi/m3/g15_raw.rs | 30 +- drivers/gpu/drm/asahi/m3/hw.rs | 18 +- drivers/gpu/drm/asahi/m3/hw/agx3.rs | 2 +- drivers/gpu/drm/asahi/m3/initdata.rs | 346 +------------------- drivers/gpu/drm/asahi/m3/m3_adt_config.rs | 146 +++------ drivers/gpu/drm/asahi/m3/m3_board.rs | 27 +- drivers/gpu/drm/asahi/m3/m3_client.rs | 40 +-- drivers/gpu/drm/asahi/m3/m3_completion.rs | 8 +- drivers/gpu/drm/asahi/m3/m3_compute.rs | 49 +-- drivers/gpu/drm/asahi/m3/m3_config.rs | 123 +++----- drivers/gpu/drm/asahi/m3/m3_device.rs | 117 +++---- drivers/gpu/drm/asahi/m3/m3_drm.rs | 197 +++++++----- drivers/gpu/drm/asahi/m3/m3_firmware.rs | 57 +--- drivers/gpu/drm/asahi/m3/m3_memory.rs | 54 +++- drivers/gpu/drm/asahi/m3/m3_params.rs | 341 ++++---------------- drivers/gpu/drm/asahi/m3/m3_pass.rs | 6 +- drivers/gpu/drm/asahi/m3/m3_pool.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_render.rs | 73 ++--- drivers/gpu/drm/asahi/m3/m3_resources.rs | 12 +- drivers/gpu/drm/asahi/m3/m3_rtkit.rs | 46 +-- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 288 +++++++++-------- drivers/gpu/drm/asahi/m3/m3_soc.rs | 367 +++++++++++++++++++--- drivers/gpu/drm/asahi/m3/m3_submit.rs | 48 +-- drivers/gpu/drm/asahi/m3/m3_thermal.rs | 17 +- drivers/gpu/drm/asahi/m3/mod.rs | 94 +++++- drivers/gpu/drm/asahi/m3/t8122_knobs.rs | 2 +- drivers/gpu/drm/asahi/m3/t8122_start.rs | 22 +- drivers/gpu/drm/asahi/mmu.rs | 2 + drivers/gpu/drm/asahi/mmu/m3.rs | 142 +++++++++ rust/kernel/soc/apple/rtkit.rs | 12 + rust/kernel/sync/completion.rs | 22 ++ 41 files changed, 1467 insertions(+), 1486 deletions(-) delete mode 100644 drivers/gpu/drm/asahi/m3/agx_fault.rs create mode 100644 drivers/gpu/drm/asahi/mmu/m3.rs diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 958600a91b06bd..4cfe26f28b8918 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -36,6 +36,21 @@ config DRM_ASAHI_DEBUG_ALLOCATOR Say N unless you are debugging firmware structures or porting to a new firmware version. +config DRM_ASAHI_M3_T8122 + bool "M3 MacBook Air GPU runtime (T8122, 14.8.3 firmware)" + depends on DRM_ASAHI + # The runtime adopts the firmware's UAT through the G15 backend's handoff helpers. + depends on DRM_ASAHI_G15 + select CRYPTO_LIB_SHA256 + default n + help + Build the M3 runtime that starts the GPU of the M3 MacBook Air + (J613, J615) on the 14.8.3 GPU firmware the boot loader leaves + loaded. It starts only with asahi.t8122_start=1 on the kernel + command line, after the boot loader handed the GPU over; without + it the T8122 probe is refused as before. Other GPUs are not + affected. + config DRM_ASAHI_G15 bool "Native G15 firmware backend (M3 Pro/Max)" depends on DRM_ASAHI diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs index 50628ade5ab8e4..e8087442d5320e 100644 --- a/drivers/gpu/drm/asahi/debug.rs +++ b/drivers/gpu/drm/asahi/debug.rs @@ -54,6 +54,13 @@ pub(crate) enum DebugFlags { DeviceControlCh = 21, FwCtlCh = 22, + // Host submission/completion timestamps without verbose firmware tracing. + SubmitTiming = 23, + // Low-volume M3 command/geometry totals without per-IRQ printk. + M3SubmitSummary = 24, + // Per-slot retired timestamps, without enabling firmware ktrace. + M3PassTiming = 25, + // 32-35: Allocator debugging FillAllocations = 32, DebugAllocations = 33, diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index c092595180cf89..43fa17412815d8 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -49,6 +49,8 @@ pub(crate) struct AsahiPlatformDriverData<'bound> { _reg: drm::Registration<'bound, AsahiDrmDriver>, /// The G17 GPU, whose coprocessors must be stopped when the driver is unbound. g17: Option>, + /// The M3 T8122 runtime, stopped on unbind like the G17 GPU. + m3: Option, /// Raw `struct device *` of the platform device. Stored once at probe /// time so the `Drop` impl can find the sysfs file on unregister. raw_dev: *mut bindings::device, @@ -62,6 +64,11 @@ unsafe impl Send for AsahiPlatformDriverData<'_> {} // SAFETY: As above. unsafe impl Sync for AsahiPlatformDriverData<'_> {} +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +type M3Backend = Arc; +#[cfg(not(CONFIG_DRM_ASAHI_M3_T8122))] +type M3Backend = core::convert::Infallible; + /// Holds a reference to the top-level GPU object. #[pin_data] pub(crate) struct AsahiDrmRegistrationData<'drm> { @@ -213,6 +220,11 @@ impl platform::Driver for AsahiPlatformDriver { // Driver data is dropped after devres, so CPU control must happen in unbind. gpu.shutdown(); } + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + if let Some(m3) = this.m3.as_ref() { + // As for G17: the runtime stops its coprocessor through device resources. + m3.stop(); + } } /// Device probe function. @@ -227,6 +239,8 @@ impl platform::Driver for AsahiPlatformDriver { let cfg = match info.ok_or(ENODEV)? { ProbeConfig::Legacy(cfg) => *cfg, ProbeConfig::G17(cfg) => return AsahiPlatformDriver::probe_g17(pdev, cfg), + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + ProbeConfig::M3(crate::m3::Chip::T8122) => return AsahiPlatformDriver::probe_t8122(pdev), ProbeConfig::M3(chip) => return Err(crate::m3::refuse_probe(pdev, *chip)), }; @@ -297,6 +311,7 @@ impl platform::Driver for AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, raw_dev, raw_stats, stats, @@ -328,6 +343,43 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: None, + m3: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }) + } + + /// The M3 MacBook Air (T8122): the M3 runtime, started only when its identity gate, the + /// backend selection and the start experiment (`asahi.t8122_start=1`) admit it; otherwise + /// the probe is refused as without the runtime. + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + fn probe_t8122<'bound>( + pdev: &'bound platform::Device>, + ) -> Result> { + let soc = crate::m3::t8122_admitted(pdev)?; + // SAFETY: The admitted M3 GPU's UAT outputs 42-bit physical addresses. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; + let resources = regs::Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let backend = crate::m3::m3_drm::Backend::start(pdev, &drm, soc)?; + let gpu = backend.clone() as Arc; + let data = try_pin_init!(AsahiDrmRegistrationData { gpu, resources }); + // SAFETY: The registration is stored and unregistered by platform data destruction, + // which drops the runtime with it. + let reg = match unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0) } { + Ok(reg) => reg, + Err(error) => { + backend.stop(); + return Err(error); + } + }; + dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); + Ok(AsahiPlatformDriverData { + _device: reg.device().into(), + _reg: reg, + g17: None, + m3: Some(backend), raw_dev: core::ptr::null_mut(), raw_stats: core::ptr::null_mut(), stats: None, @@ -371,6 +423,7 @@ impl AsahiPlatformDriver { _device: reg.device().into(), _reg: reg, g17: Some(gpu), + m3: None, raw_dev, raw_stats, stats: None, @@ -390,5 +443,9 @@ impl Drop for AsahiPlatformDriverData<'_> { // Also covers failure to allocate platform driver data after the probe body returned. gpu.shutdown(); } + #[cfg(CONFIG_DRM_ASAHI_M3_T8122)] + if let Some(m3) = self.m3.as_ref() { + m3.stop(); + } } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 608864949c976c..242f6cdcd3f838 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -233,6 +233,9 @@ pub(crate) struct File { queues: xarray::XArray>>>, // #[pin] objects: xarray::XArray>, + /// Largest size this file has asked GET_PARAMS for the global parameters with + /// ([`gpu::Gpu::admit_vm`]). + params_size: core::sync::atomic::AtomicU64, } /// Convenience type alias for our DRM `File` type. @@ -286,6 +289,7 @@ impl File { (*slot).id = id; (*slot).user_range = user_range; + addr_of_mut!((*slot).params_size).write(core::sync::atomic::AtomicU64::new(0)); Ok(()) }) } @@ -386,6 +390,9 @@ impl File { core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) })?; + file.inner() + .params_size + .fetch_max(data.size, core::sync::atomic::Ordering::Relaxed); Ok(0) } @@ -454,6 +461,13 @@ impl File { // Every other runtime here reports DRM_ASAHI_GPU_HAL_LEGACY. return Err(EINVAL); } + reg_data.gpu.admit_vm( + device.as_ref().as_ref(), + file.inner() + .params_size + .load(core::sync::atomic::Ordering::Relaxed), + data.flags, + )?; let kernel_range = data.kernel_start..data.kernel_end; let user_range = file.inner().user_range.clone(); diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs index 46c05699d77225..a740ad3bb945df 100644 --- a/drivers/gpu/drm/asahi/float.rs +++ b/drivers/gpu/drm/asahi/float.rs @@ -49,6 +49,11 @@ impl F32 { F32(u) } + /// Convert to the raw IEEE 754 bits. + pub(crate) const fn to_bits(self) -> u32 { + self.0 + } + // Convert a `f32` value into an F32 // // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs index 75d3ecc30ffe5a..324bc08194e531 100644 --- a/drivers/gpu/drm/asahi/g15/mod.rs +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -68,10 +68,10 @@ mod pools; mod power; mod queue; mod recovery; -mod regs; +pub(crate) mod regs; mod slots; mod timestamp; -mod uat; +pub(crate) mod uat; mod vm; mod wire; diff --git a/drivers/gpu/drm/asahi/g15/regs.rs b/drivers/gpu/drm/asahi/g15/regs.rs index d5435e9f1c7eaa..05a1b390b3dffc 100644 --- a/drivers/gpu/drm/asahi/g15/regs.rs +++ b/drivers/gpu/drm/asahi/g15/regs.rs @@ -152,6 +152,21 @@ impl Mapping { pub(crate) fn phys(&self) -> u64 { self.phys } + + /// Cleans the whole mapping to the point of coherency when it is the kernel's cacheable + /// mapping of reserved System RAM ([`Self::region`]); device and write-combined mappings + /// need nothing. + pub(crate) fn clean_poc(&self) { + if self.kind != Kind::WriteBack { + return; + } + let start = self.raw.addr(); + // SAFETY: `start..start + maxsize` is this live mapping; cleaning a cacheable range has + // no other effect. + unsafe { + bindings::dcache_clean_poc(start as _, (start + self.raw.size()) as _); + } + } } impl Drop for Mapping { diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs index 854ff34e729a37..3855bd0beb9cb6 100644 --- a/drivers/gpu/drm/asahi/g15/uat.rs +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -200,16 +200,41 @@ pub(crate) struct Window { impl Window { /// Maps the UAT handoff regions ([`Mapping::region`]). pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { + Self::from_regions(adt.handoff.start, adt.ttbs.start, adt.tables.start, adt.shared_l2.start) + } + + /// Maps the UAT handoff regions at the given physical addresses: the handoff page, the + /// TTBAT, the firmware kernel root and the shared L2 page. The M3 T8122 runtime takes them + /// from its admitted reserved-memory regions. + pub(crate) fn from_regions(handoff: u64, ttbs: u64, root: u64, shared_l2: u64) -> Result { Ok(Self { - handoff: Mapping::region(adt.handoff.start, hw::HANDOFF_SIZE as usize)?, - ttbs: Mapping::region(adt.ttbs.start, hw::TTBS_SIZE as usize)?, - root: Mapping::region(adt.tables.start, hw::PAGE as usize)?, - shared_l2: Mapping::region(adt.shared_l2.start, hw::SHARED_L2_SIZE as usize)?, + handoff: Mapping::region(handoff, hw::HANDOFF_SIZE as usize)?, + ttbs: Mapping::region(ttbs, hw::TTBS_SIZE as usize)?, + root: Mapping::region(root, hw::PAGE as usize)?, + shared_l2: Mapping::region(shared_l2, hw::SHARED_L2_SIZE as usize)?, image: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, snapshot: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, }) } + /// [`Self::sync_range`] over the whole host window, then [`Self::clean_tables`]. + pub(crate) fn sync_all(&mut self, uat: &mmu::Uat) -> Result { + self.sync_inner(uat, None)?; + self.clean_tables(); + Ok(()) + } + + /// Cleans the firmware-walked pages this window writes (the kernel root, the shared L2, + /// the TTBAT and the handoff page) to the point of coherency. Only a region that is + /// reserved System RAM, mapped cacheable, has anything to clean: the UAT walker does not + /// snoop the CPU caches. The M3 T8122 runtime calls it after [`Self::bootstrap`]. + pub(crate) fn clean_tables(&self) { + for map in [&self.root, &self.shared_l2, &self.ttbs, &self.handoff] { + map.clean_poc(); + } + mb(); + } + /// Cold-start check: all 128 TTBAT words are zero. pub(crate) fn ttbat_is_clear(&self) -> Result { let t = self.ttbs.io(); diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 90bb25451019e7..b9283efb470a93 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -338,6 +338,12 @@ pub(crate) trait Gpu: Send + Sync { } /// A userspace VM handle was closed; release backend-only registry references. fn vm_closed(&self) {} + /// Whether a file may create a VM: `params_size` is the largest size it asked GET_PARAMS + /// for the global parameters with, `flags` its VM_CREATE flags. Every GPU but the M3 + /// T8122 runtime admits any client. + fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { + Ok(()) + } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. diff --git a/drivers/gpu/drm/asahi/m3/agx_fault.rs b/drivers/gpu/drm/asahi/m3/agx_fault.rs deleted file mode 100644 index 9c37cfa98dce9f..00000000000000 --- a/drivers/gpu/drm/asahi/m3/agx_fault.rs +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Immutable fault-only snapshots of owned M3 firmware memory. -//! -//! Records contain name[32], firmware VA, length, begin/end monotonic ns, -//! then exactly `length` bytes. No MMIO or application BOs are read. Reads -//! are sequential observations, not atomic with running firmware. - -use kernel::{prelude::*, time::{ClockSource, Monotonic}}; - -pub(crate) struct Dump { - bytes: KVVec, - count: u32, - limit: usize, -} - -impl Dump { - - /// M3 snapshots have no scheduler job ID yet and contain only bounded - /// driver-owned firmware configuration and channel storage. - pub(crate) fn new_m3() -> Result { - Self::with_format(0, b"M3FWD001", 2 * 1024 * 1024) - } - - fn with_format(job: u64, magic: &[u8; 8], limit: usize) -> Result { - // Reserve before firmware/job publication, while healthy. Record appends - // must never enter reclaim waiting on the GPU that just faulted. - let mut bytes = KVVec::with_capacity(limit, GFP_KERNEL)?; - bytes.resize(40, 0, GFP_KERNEL)?; - bytes[..8].copy_from_slice(magic); - bytes[16..24].copy_from_slice(&job.to_le_bytes()); - bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); - Ok(Self { bytes, count: 0, limit }) - } - - /// Start the one-shot capture; construction may precede the fault by hours. - pub(crate) fn begin(&mut self, job: u64) { - self.bytes[16..24].copy_from_slice(&job.to_le_bytes()); - self.bytes[24..32].copy_from_slice(&(Monotonic::ktime_get() as u64).to_le_bytes()); - } - - pub(crate) fn record(&mut self, name: &str, va: u64, size: usize, - read: impl FnOnce(&mut [u8]) -> Result) -> Result { - if name.is_empty() || name.len() >= 32 || size == 0 { return Err(EINVAL); } - let count = self.count.checked_add(1).ok_or(EOVERFLOW)?; - let start = self.bytes.len(); - let end = start.checked_add(64).and_then(|x| x.checked_add(size)).ok_or(EOVERFLOW)?; - if end > self.limit { return Err(E2BIG); } - // Capacity was reserved at construction. NOWAIT is defense in depth; - // the checked limit above guarantees resize cannot allocate here. - self.bytes.resize(end, 0, GFP_NOWAIT)?; - self.bytes[start..start+name.len()].copy_from_slice(name.as_bytes()); - self.bytes[start+32..start+40].copy_from_slice(&va.to_le_bytes()); - self.bytes[start+40..start+48].copy_from_slice(&(size as u64).to_le_bytes()); - let begin = Monotonic::ktime_get() as u64; - if let Err(error) = read(&mut self.bytes[start+64..end]) { - self.bytes.truncate(start); - return Err(error); - } - let finish = Monotonic::ktime_get() as u64; - self.bytes[start+48..start+56].copy_from_slice(&begin.to_le_bytes()); - self.bytes[start+56..start+64].copy_from_slice(&finish.to_le_bytes()); - self.count = count; - Ok(()) - } - - pub(crate) fn publish(mut self, dev: &kernel::device::Device) -> Result { - let length = self.bytes.len(); - self.bytes[8..12].copy_from_slice(&self.count.to_le_bytes()); - self.bytes[12..16].copy_from_slice(&1u32.to_le_bytes()); - self.bytes[32..40].copy_from_slice(&(length as u64).to_le_bytes()); - let format = "M3FWD001"; - let owned = KBox::new(self, GFP_NOWAIT)?; - kernel::devcoredump::dev_coredump(dev, &crate::THIS_MODULE, owned, GFP_NOWAIT, - kernel::devcoredump::DEFAULT_TIMEOUT); - dev_info!(dev, "Asahi: sealed {}-byte firmware fault snapshot offered to devcoredump ({})\n", length, format); - Ok(()) - } -} - -impl kernel::devcoredump::DevCoreDump for Dump { - fn read(&self, output: &mut [u8], offset: usize) -> Result { - if offset >= self.bytes.len() { return Ok(0); } - let length = output.len().min(self.bytes.len()-offset); - output[..length].copy_from_slice(&self.bytes[offset..offset+length]); - Ok(length) - } -} diff --git a/drivers/gpu/drm/asahi/m3/g15_raw.rs b/drivers/gpu/drm/asahi/m3/g15_raw.rs index a876cad1ec283c..785b7f2314c8f4 100644 --- a/drivers/gpu/drm/asahi/m3/g15_raw.rs +++ b/drivers/gpu/drm/asahi/m3/g15_raw.rs @@ -2,8 +2,8 @@ //! GPU initialization / global structures -use super::channels; -use super::types::*; +use crate::fw::channels; +use crate::fw::types::*; use crate::{ default_zeroed, gem, @@ -1158,7 +1158,9 @@ pub(crate) mod raw { pub(crate) event: ChannelRing, pub(crate) fw_log: ChannelRing, pub(crate) ktrace: ChannelRing, - pub(crate) stats: ChannelRing, + // The G15 stats record has the 13.x size (STATS_SZ for V >= V13_0B4); 7.2 generates + // only the 13.x instances. + pub(crate) stats: ChannelRing, /// FWLog payload buffer (0x600 x 0xd8). +0x1f8 @@ -1548,7 +1550,7 @@ impl GpuStruct for InitDataG15V14_8_3 { // G15 / 14.8.3 firmware structure layout checks: the sizes and offsets the firmware expects. mod g15_layout { - use super::channels; + use crate::fw::channels; use super::raw::*; use core::mem::{offset_of, size_of}; use kernel::static_assert; @@ -1602,7 +1604,7 @@ mod g15_layout { static_assert!(size_of::() == 0xe00); static_assert!(size_of::() == 0x18); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_28) == 0x28); - static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3_globals_debug::OFFSET); + static_assert!(offset_of!(GlobalsG15V14_8_3, debug) == crate::m3::m3_globals_debug::OFFSET); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_38) == 0x38); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_50) == 0x50); static_assert!(offset_of!(GlobalsG15V14_8_3, unk_56) == 0x56); @@ -1778,23 +1780,5 @@ mod g15_layout { == 0xc30 ); - // G13/G14X 13.5 sizes must not change (regression guard for the per-field gating). - // RuntimePointers: gpu_scratch at 0x280 (0x270 on G14X) + RuntimeScratch; note that these - // are 0x20 more than ver_info word 0 (0x6ba0/0xb390). The others are the current G13/G14X - // sizes. - static_assert!(size_of::>() == 0x6bc0); - static_assert!(size_of::>() == 0xb3b0); - static_assert!(size_of::() == 0x12394); - static_assert!(size_of::() == 0x1715c); - static_assert!(size_of::() == 0x421c); - static_assert!(size_of::() == 0x6c34); - static_assert!(size_of::() == 0x1884); - static_assert!(size_of::() == 0x1884); - static_assert!(size_of::>() == 0xbc); - static_assert!(size_of::>() == 0xbc); - static_assert!(size_of::>() == 0xb4); - static_assert!(size_of::() == 0x30); - static_assert!(size_of::() == 0x1124); - static_assert!(size_of::() == 0x1a34); static_assert!(size_of::>() == 0x30); } diff --git a/drivers/gpu/drm/asahi/m3/hw.rs b/drivers/gpu/drm/asahi/m3/hw.rs index c90156bd1244bd..a774f5de6573f5 100644 --- a/drivers/gpu/drm/asahi/m3/hw.rs +++ b/drivers/gpu/drm/asahi/m3/hw.rs @@ -5,21 +5,18 @@ //! This module contains the definitions used to store per-GPU and per-SoC configuration data. use crate::fw::types::*; -pub(crate) use crate::identity::{GpuGen, GpuHalGeneration, GpuVariant}; +pub(crate) use crate::m3::identity::{GpuGen, GpuHalGeneration, GpuVariant}; use kernel::c_str; use kernel::prelude::*; const MAX_POWERZONES: usize = 5; +/// G15 SRAM rail floor for a device tree without `apple,min-sram-microvolt`, in microvolts: the +/// J516S runtime ADT perf-states-sram table floors SRAM at 810 mV (7.1.12 `hw::t6030`). +const G15_MIN_SRAM_MICROVOLT: u32 = 810_000; + pub(crate) mod agx3; -pub(crate) mod t600x; -pub(crate) mod t602x; -pub(crate) mod t6030; -pub(crate) mod t6031; -pub(crate) mod t8103; -pub(crate) mod t8112; pub(crate) mod t8122; -pub(crate) mod t8140; /// GPU revision enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] @@ -559,6 +556,9 @@ impl PwrConfig { return Err(EINVAL); } + // Kept as on the 7.1.12 kernel this runtime was qualified with: it steps by 3 over the + // zone count, which reads the same zones as one zone per triple for up to one zone and + // fewer beyond. (7.2's shared copy reads one zone per triple.) let pz_count = pz_data.len() / 3; let mut power_zones = KVec::new(); for i in (0..pz_count).step_by(3) { @@ -610,7 +610,7 @@ impl PwrConfig { min_sram_microvolt: if cfg.gpu_gen == GpuGen::G15 { // SRAM rail floor: 810 mV from the J516S runtime ADT perf-states-sram table. // Tolerate a DT without the property on G15 only. - prop!("apple,min-sram-microvolt", t6030::MIN_SRAM_MICROVOLT) + prop!("apple,min-sram-microvolt", G15_MIN_SRAM_MICROVOLT) } else { prop!("apple,min-sram-microvolt") }, diff --git a/drivers/gpu/drm/asahi/m3/hw/agx3.rs b/drivers/gpu/drm/asahi/m3/hw/agx3.rs index cb0c4fdbfa0126..f5e6ad607ec25d 100644 --- a/drivers/gpu/drm/asahi/m3/hw/agx3.rs +++ b/drivers/gpu/drm/asahi/m3/hw/agx3.rs @@ -2,7 +2,7 @@ #[cfg(not(test))] -use crate::identity::{GpuGen, GpuVariant}; +use crate::m3::identity::{GpuGen, GpuVariant}; #[cfg(test)] #[path = "../identity.rs"] mod identity; diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs index dfeffd900eb12b..2348d075a731cf 100644 --- a/drivers/gpu/drm/asahi/m3/initdata.rs +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -12,21 +12,16 @@ //! derived from observing the InitData structures that macOS generates. use crate::f32; -use crate::fw::initdata::*; +use crate::m3::g15_raw::*; use crate::fw::types::*; -use crate::module_parameters; use crate::{ - driver::AsahiDevice, - gem, - gpu, - hw, + m3::hw, mmu, // }; use kernel::error::{ Error, Result, // }; -use kernel::macros::versions; use kernel::prelude::*; use kernel::try_init; @@ -35,6 +30,10 @@ use ::pin_init::Init; /// Builder helper for the global GPU InitData. +/// The "unknown page" of the 39-bit user layout (7.1.12 `mmu::IOVA_UNK_PAGE`): two pages below +/// the top of the user range. +const IOVA_UNK_PAGE: u64 = (1 << 39) - 2 * 0x4000; + /// G15 idle-off delay (ms, Globals +0x9a0/+0x9a4) used as "never idle off" while the rings live /// in Fender scratch SRAM without a backup/restore path. /// TODO: the firmware's encoding of "never" is unknown. i32::MAX stays positive if the @@ -104,24 +103,6 @@ pub(crate) struct G15RuntimeHwDataB { } impl G15Options { - /// The options of the G15 manager backend: its validated defaults, the performance-state - /// cap (`asahi.m3_max_pstate`) and the `asahi.g15_debug` A/B switches. - fn manager(cfg: &'static hw::HwConfig) -> G15Options { - let g15 = cfg.gpu_gen == hw::GpuGen::G15; - G15Options { - layout: G15Layout::Manager, - split_pstates: g15 - && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerSplitPstates), - cap: g15.then(|| { - crate::m3_params::max_pstate_param().unwrap_or(crate::m3_params::MAX_PSTATE_DEFAULT) - }), - timestamp_base: None, - reference_ppm: g15 - && crate::m3_params::g15_debug(crate::m3_params::G15Debug::ManagerReferencePpm), - runtime_hwdata_b: None, - per_cluster_voltages: false, - } - } /// The published performance-state tables for `pwr`. pub(crate) fn tables(&self, pwr: &hw::PwrConfig) -> Result { @@ -318,43 +299,12 @@ impl PStateTables { } } -/// The highest GPU frequency the G15 manager backend lets the firmware use, in kHz (its -/// performance cap), or None when the GPU is not G15 or the cap cannot be applied. -pub(crate) fn g15_manager_max_frequency_khz( - cfg: &'static hw::HwConfig, - pwr: &hw::PwrConfig, -) -> Option { - if cfg.gpu_gen != hw::GpuGen::G15 { - return None; - } - let tables = G15Options::manager(cfg).tables(pwr).ok()?; - Some(tables.max_frequency_khz(pwr)) -} -pub(crate) struct InitDataBuilderG15V14_8_3<'a> { - dev: &'a AsahiDevice, - alloc: &'a mut gpu::KernelAllocators, - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - g15: G15Options, -} +/// The G15 InitData builders. The M3 runtime places the firmware objects itself, so only the +/// builders into plain memory are kept ([`Self::g15_contents`]). +pub(crate) struct InitDataBuilderG15V14_8_3<'a>(core::marker::PhantomData<&'a ()>); impl<'a> InitDataBuilderG15V14_8_3<'a> { - /// Create a new InitData builder - pub(crate) fn new( - dev: &'a AsahiDevice, - alloc: &'a mut gpu::KernelAllocators, - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - ) -> InitDataBuilderG15V14_8_3<'a> { - InitDataBuilderG15V14_8_3 { - dev, - alloc, - cfg, - dyncfg, - g15: G15Options::manager(cfg), - } - } /// Create an unknown T81xx-specific data structure. fn t81xx_data( @@ -591,47 +541,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { })) } - /// Create the HwDataA structure. This mostly contains power-related configuration. - fn hwdata_a(&mut self) -> Result> { - let init = Self::hwdata_a_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - if !dyncfg.hw_data_b.is_empty() { - unsafe { - let mut matches: bool = true; - let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataAG15V14_8_3 as *const u8, - core::mem::size_of::(), - ); - if sla.len() != dyncfg.hw_data_a.len() { - matches = false; - dev_err!( - dev.as_ref(), - "!!! Hwdata A size mismatch: {} {}", - sla.len(), - dyncfg.hw_data_a.len(), - ); - } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { - if sla[i] != dyncfg.hw_data_a[i] { - matches = false; - dev_err!(dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); - break; - } - } - if matches { - dev_info!(dev.as_ref(), "!!! Hwdata A match"); - } - } - } - Ok(()) - }) - }) - } /// The runtime backend's HwDataA words that the manager layout leaves at their defaults: the /// shader-engine controller block at +0x10b4 in its G15 order (all inputs from the device @@ -698,30 +607,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { raw.unk_429c[0] = 1; } - // Compare a structure against the copy m1n1 left in the reserved-memory blob, if any - // (debug aid, same as the G13/G14 builders do inline). - fn compare_blob(dev: &AsahiDevice, what: &str, raw: &T, blob: &[u8]) { - if blob.is_empty() { - return; - } - // SAFETY: `raw` is a fully initialized firmware structure; viewing it as bytes is fine. - let sla = unsafe { - core::slice::from_raw_parts(raw as *const T as *const u8, core::mem::size_of::()) - }; - if sla.len() != blob.len() { - dev_err!( - dev.as_ref(), - "!!! {} size mismatch: {} {}", - what, - sla.len(), - blob.len() - ); - } - match sla.iter().zip(blob.iter()).position(|(a, b)| a != b) { - Some(i) => dev_err!(dev.as_ref(), "!!! {} first mismatch: {i}", what), - None => dev_info!(dev.as_ref(), "!!! {} match", what), - } - } // Build the G15 (14.8.3) HwDataB contents without allocating the object. Board values whose // source is not known yet are left 0 and marked TODO. @@ -737,7 +622,8 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { let runtime_hwdata_b = g15.runtime_hwdata_b; let timestamp_base = match g15.timestamp_base { Some(base) => base, - None => mmu::kern_iova(cfg, gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + // The runtime always names its timestamp arena. + None => return Err(EINVAL), }; Ok(try_init!(raw::HwDataBG15V14_8_3 { // VA-layout words, +0x0/+0x8 @@ -750,7 +636,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { // "Unknown page": the G15 firmware layout uses 0x2ff_ffff8000 (+0x20) in its // 42-bit user layout; the manager keeps a 39-bit user range and maps the page // at IOVA_UNK_PAGE, so it passes that. - unk_30: U64(mmu::IOVA_UNK_PAGE), + unk_30: U64(IOVA_UNK_PAGE), // FW VA of the timestamp area, +0x28 timestamp_area_base: U64(timestamp_base), // TODO: CSC matrices (+0x38) and border-colour pointer (+0x638) @@ -960,20 +846,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { raw.unk_1818 = words.unk_1818; } - // Create the G15 (14.8.3) HwDataB structure. - fn hwdata_b(&mut self) -> Result> { - let init = Self::hwdata_b_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - Self::compare_blob(dev, "Hwdata B", &*raw, &dyncfg.hw_data_b); - Ok(()) - }) - }) - } // Build the G15 (14.8.3) Globals contents without allocating the object. Only the fields // with a known init value are set; the rest start at 0. @@ -1102,123 +974,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { Ok(()) } - // Create the G15 (14.8.3) Globals structure. - fn globals(&mut self) -> Result> { - let init = Self::globals_init(self.cfg, self.dyncfg, &self.g15)?; - let dev = self.dev; - let dyncfg = self.dyncfg; - self.alloc - .private - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| { - init.chain(move |raw| { - Self::compare_blob(dev, "Globals", &*raw, &dyncfg.hw_globals); - Ok(()) - }) - }) - } - // Create the G15 (14.8.3) RuntimePointers structure. The ring descriptors are filled in by - // the GPU manager. - fn runtime_pointers(&mut self) -> Result> { - let hwa = self.hwdata_a()?; - let hwb = self.hwdata_b()?; - let bufmgr_fw_addr = mmu::kern_iova(self.cfg, gpu::IOVA_KERN_GPU_BUFMGR_HIGH); - let rp_2d0 = self.g15.runtime_pointers_2d0(); - - let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; - buffer_mgr_ctl.vmap()?.memset(0); - - GpuObject::new_init_prealloc( - self.alloc.private.alloc_object()?, - |_ptr| { - let alloc = &mut *self.alloc; - try_init!(RuntimePointersG15V14_8_3 { - // G15 E1/E2/E3 are 0xc10/0x1248/0xe10 bytes; the G13 - // GpuGlobalStatsVtx/GpuStatsComp buffers are larger, which is harmless. - stats <- { - let alloc = &mut *alloc; - try_init!(StatsG15V14_8_3 { - vtx: alloc.private.new_default::()?, - frag: alloc.private.new_init( - pin_init::init_zeroed::(), - |_inner, _ptr| { - try_init!(raw::GpuGlobalStatsFragG15V14_8_3 { - total_cmds: 0, - unk_4: 0, - stats: Default::default(), - }) - } - )?, - comp: alloc.private.new_default::()?, - }) - }, - - hwdata_a: hwa, - hwdata_b: hwb, - - // BRN (erratum) table. - // TODO: size and contents unknown; one zeroed page (no errata). - brn_table: alloc.private.array_empty_tagged(0x4000, b"IBRN")?, - // E7: 0x88 bytes, zeroed - unkptr_e7: alloc.private.array_empty_tagged(0x88, b"IE7_")?, - // E5: 0x60 bytes, zeroed - unkptr_e5: alloc.private.array_empty_tagged(0x60, b"IE5_")?, - // UMA page-pool descriptor table (RuntimePointers +0x2c0/+0x2c8). - // TODO: size/format unknown; one zeroed page. The firmware takes a - // GPU VA and a FW VA; both point at this kernel mapping for now. - uma_pool_desc: alloc.shared.array_empty_tagged(0x4000, b"IUMA")?, - - buffer_mgr_ctl, - buffer_mgr_ctl_low_mapping: None, - buffer_mgr_ctl_high_mapping: None, - }) - }, - |inner, _ptr| { - try_init!(raw::RuntimePointersG15V14_8_3 { - hwdata_b: inner.hwdata_b.gpu_pointer(), - brn_table: inner.brn_table.gpu_pointer(), - unkptr_e7: inner.unkptr_e7.gpu_pointer(), - - pipes: Default::default(), - device_control: Default::default(), - event: Default::default(), - fw_log: Default::default(), - ktrace: Default::default(), - stats: Default::default(), - fwlog_buf: None, - __pad_200: Default::default(), - // FWLog enable: the firmware writes no log records while this is 0. - // Opt-in via asahi.g15_debug bit 43. - unk_230: crate::m3_params::g15_debug(crate::m3_params::G15Debug::FwLog) as u32, - - stats_vtx: inner.stats.vtx.gpu_pointer(), - stats_frag: inner.stats.frag.gpu_pointer(), - stats_comp: inner.stats.comp.gpu_pointer(), - unkptr_e5: inner.unkptr_e5.gpu_pointer(), - __pad_254: Default::default(), - // Optional pointer, left 0 - unkptr_2a8: U64(0), - buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), - buffer_mgr_ctl_fw_addr: U64(bufmgr_fw_addr), - uma_pool_desc_gpu_addr: U64(inner.uma_pool_desc.gpu_va().get()), - uma_pool_desc_fw_addr: U64(inner.uma_pool_desc.gpu_va().get()), - // TODO: board configuration words; +0x2d4 left 0. - unk_2d0: rp_2d0, - unk_2d4: 0, - __pad_2d8: Default::default(), - // +0x3b0: 0xff, then 0x90 zero bytes - cswitch_hist_ctl: 0xff, - unk_3b1: Default::default(), - hwdata_a: inner.hwdata_a.gpu_pointer(), - // TODO: 16 board configuration bytes; left 0. - unk_449: Default::default(), - __pad_459: Default::default(), - unk_46d: Default::default(), - __pad_48d: Default::default(), - }) - }, - ) - } // Fill the G15 P0 status block. Host-initialised words are 0 except +0x45c4, which arms the // firmware work-scheduler callbacks, and in the runtime layout the noise-suppression idle @@ -1230,13 +986,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { } } - // Create the G15 P0 status block. - fn status_block(&mut self) -> Result> { - let g15 = self.g15; - let mut block = self.alloc.shared.new_default::()?; - block.with_mut(|raw, _inner| Self::status_block_fill(raw, &g15)); - Ok(block) - } // Build the G15 P1 power-controller block without allocating it. The fields keep the order // and the formulas of the G13 13.5 Globals power fields (inferred). @@ -1306,18 +1055,7 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { }) } - // Create the G15 P1 power-controller block. - fn power_ctl_block(&mut self) -> Result> { - let init = Self::power_ctl_block_init(self.cfg, self.dyncfg, &self.g15); - self.alloc - .shared - .new_init(pin_init::init_zeroed(), move |_inner, _ptr| init) - } - // Create the G15 P2 debug block. ktrace stays disabled, as on G13. - fn debug_block(&mut self) -> Result> { - self.alloc.shared.new_default::() - } /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. fn uat_level_info( @@ -1349,66 +1087,6 @@ impl<'a> InitDataBuilderG15V14_8_3<'a> { ] } - /// Build the top-level InitData object. - #[inline(never)] - pub(crate) fn build(&mut self) -> Result>> { - { - let pwr = &self.dyncfg.pwr; - let tables = self.g15.tables(pwr)?; - dev_info!( - self.dev.as_ref(), - "G15: performance cap {} -> {} state {} of 1..={} ({} MHz)\n", - self.g15.cap.unwrap_or(0), - if tables.secondary.is_some() { "voltage-sorted" } else { "device-tree" }, - tables.max, - tables.len - 1, - tables.max_frequency_khz(pwr) / 1000 - ); - } - let runtime_pointers = self.runtime_pointers()?; - let globals = self.globals()?; - // 14.x: FwStatus lives inside the P0 status block - let fw_status = self.status_block()?; - let power_ctl_block = self.power_ctl_block()?; - let debug_block = self.debug_block()?; - let shared_ro = &mut self.alloc.shared_ro; - - let obj = self.alloc.private.new_init( - try_init!(InitDataG15V14_8_3 { - unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, - runtime_pointers, - globals, - fw_status, - power_ctl_block, - debug_block, - }), - |inner, _ptr| { - let cfg = &self.cfg; - try_init!(raw::InitDataG15V14_8_3 { - ver_info: Array::new([0x0490, 0x8380, 0xe21e, 0x0c08]), - unk_buf: inner.unk_buf.gpu_pointer(), - unk_8: 0, - unk_c: 0, - runtime_pointers: inner.runtime_pointers.gpu_pointer(), - globals: inner.globals.gpu_pointer(), - // InitData +0x28/+0x2c: u32 0, u32 1 - unk_28: 0, - unk_2c: 1, - uat_page_size: 0x4000, - uat_page_bits: 14, - uat_num_levels: 3, - uat_level_info: Array::new(Self::uat_levels(cfg)), - // TODO: pmap descriptor +0x7c, meaning unknown; left 0. - unk_94: 0, - __pad_98: Default::default(), - debug_block: inner.debug_block.gpu_pointer(), - status_block: inner.fw_status.gpu_pointer(), - power_ctl_block: inner.power_ctl_block.gpu_pointer(), - }) - }, - )?; - Ok(KBox::new(obj, GFP_KERNEL)?) - } } /// G15 InitData contents built into plain memory, for a backend that places the firmware objects diff --git a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs index 2fa384c910688e..1893c113cf2a7d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_adt_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_adt_config.rs @@ -13,18 +13,18 @@ use kernel::{ use crate::{ float::F32, - fw::initdata::raw, - hw, initdata, - initdata::{G15Layout, G15Options}, - m3_config::Records, - m3_firmware::Firmware, - m3_init_layout as init, - m3_init_storage as storage, - m3_memory::Buffer, - m3_params::{self, G15Debug, InitDataParam, ThermalMode}, - m3_soc::{IoMapping, PstateTable, Soc}, - t8122_hwdata, - t8122_start::Experiment, + m3::g15_raw::raw, + m3::hw, m3::initdata, + m3::initdata::{G15Layout, G15Options}, + m3::m3_config::Records, + m3::m3_firmware::Firmware, + m3::m3_init_layout as init, + m3::m3_init_storage as storage, + m3::m3_memory::Buffer, + m3::m3_params::{self, G15Debug, InitDataParam, ThermalMode}, + m3::m3_soc::{IoMapping, PstateTable, Soc}, + m3::t8122_hwdata, + m3::t8122_start::Experiment, }; // The T8122 HwDataA +0x418c block (`t8122_hwdata`) covers exactly the G13-era leakage @@ -275,7 +275,7 @@ pub(crate) struct Contents { /// map read-write) except in the T8122 start experiment. pub(crate) read_only_slots: u32, /// The HwData object's placement and mapping (`Soc::hwdata_object`). - pub(crate) hwdata_object: Option, + pub(crate) hwdata_object: Option, } fn read_u32(bytes: &[u8], offset: usize) -> Result { @@ -296,36 +296,27 @@ impl Contents { /// any GPU register is touched. /// /// `experiment` is the T8122 start experiment's values (`t8122_start`), on an armed T8122 - /// only; it then stands in for the SoC table's missing ones. `t6031` is the same for an - /// armed T6031 (`t6031_start`). + /// only; it then stands in for the SoC table's missing ones. pub(crate) fn select( - pdev: &platform::Device, + pdev: &platform::Device>, firmware: &Firmware, soc: &Soc, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, ) -> Result { let dev = pdev.as_ref(); let param = m3_params::initdata_param(); - check_version(dev, firmware, experiment, t6031)?; + check_version(dev, firmware, experiment)?; let iomaps = match experiment { Some(e) => e.iomaps(), - None => match t6031 { - Some(e) => e.iomaps(), - None => soc.iomaps.ok_or(ENODEV)?, - }, + None => soc.iomaps.ok_or(ENODEV)?, }; storage::validate_iomaps(iomaps).map_err(|_| EINVAL)?; - let images = build_images(dev, firmware, soc, experiment, t6031, iomaps)?; + let images = build_images(dev, firmware, soc, experiment, iomaps)?; let hwdata = images.get(HWDATA).and_then(|i| i.as_deref()).ok_or(EINVAL)?; - let pstates = PstatePolicy::new( - dev, - hwdata, - experiment.map(|e| e.pstate_cap()).or_else(|| t6031.map(|e| e.pstate_cap())), - )?; + let pstates = PstatePolicy::new(dev, hwdata, experiment.map(|e| e.pstate_cap()))?; let table = match soc.pstates { PstateTable::Fixed { states, top_mhz } => (states, top_mhz), - PstateTable::DeviceTree => crate::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { + PstateTable::DeviceTree => crate::m3::m3_board::opp_table_shape(pdev, soc).ok_or_else(|| { dev_err!( dev, "M3: the device tree's operating points give no performance-state table\n" @@ -346,7 +337,7 @@ impl Contents { return Err(EINVAL); } // The published states never go above the device tree's highest operating point. - if let Some(dt_khz) = crate::m3_board::max_frequency_khz(pdev) { + if let Some(dt_khz) = crate::m3::m3_board::max_frequency_khz(pdev) { if 1000 * pstates.max_mhz > dt_khz { dev_err!( dev, @@ -377,14 +368,8 @@ impl Contents { pstates.max, pstates.max_mhz ); - let read_only_slots = experiment - .map(|e| e.read_only_slots()) - .or_else(|| t6031.map(|e| e.read_only_slots())) - .unwrap_or(0); - let hwdata_object = match t6031 { - Some(e) => e.hwdata_object(), - None => soc.hwdata_object, - }; + let read_only_slots = experiment.map(|e| e.read_only_slots()).unwrap_or(0); + let hwdata_object = soc.hwdata_object; Ok(Contents { images, pstates, iomaps, table, read_only_slots, hwdata_object }) } @@ -435,7 +420,6 @@ fn check_version( dev: &device::Device, firmware: &Firmware, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, ) -> Result { let version = constructed_version()?; if version != firmware.initdata_magic && experiment.is_some() { @@ -447,15 +431,6 @@ fn check_version( ); return Ok(()); } - if version != firmware.initdata_magic && t6031.is_some() { - dev_warn!( - dev, - "M3 G15C start: InitData version {:#x} replaces the constructed {:#x}\n", - firmware.initdata_magic, - version - ); - return Ok(()); - } if version != firmware.initdata_magic { dev_err!( dev, @@ -572,7 +547,6 @@ fn fill_io_mappings( dev: &device::Device, cfg: &'static hw::HwConfig, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, mappings: &[IoMapping], iomaps: &[storage::IoMap], hwdata: &mut [u8], @@ -585,10 +559,9 @@ fn fill_io_mappings( let virt = entry + offset_of!(raw::IOMapping, virt_addr); // The same register block as the manager's table. let same_block = cfg.io_mappings.get(slot).and_then(|m| m.as_ref()).is_some_and(|m| { - let base = match (experiment, t6031) { - (Some(e), _) => e.io_block(slot, m.base as u64), - (None, Some(e)) => e.io_block(slot, m.base as u64), - (None, None) => m.base as u64, + let base = match experiment { + Some(e) => e.io_block(slot, m.base as u64), + None => m.base as u64, }; base & !0x3fff == phys & !0x3fff && m.writable == writable }); @@ -720,7 +693,7 @@ fn whole(v: F32) -> u32 { /// The boot loader's decoded GPU leakage fuse values (`Soc::leak_fuse`), logged when it exported /// them. Returns them when the boot loader's switch is set. With the switch set and no two /// valid values, refuses: the boot loader then used neither the fuse nor the stand-in. -fn leak_fuse(dev: &device::Device, lf: &crate::m3_soc::LeakFuse) -> Result> { +fn leak_fuse(dev: &device::Device, lf: &crate::m3::m3_soc::LeakFuse) -> Result> { let chosen = kernel::of::chosen(); let switch = chosen .as_ref() @@ -790,29 +763,19 @@ fn build_images( firmware: &Firmware, soc: &Soc, experiment: Option<&Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, iomaps: &[storage::IoMap], ) -> Result>>> { check_layout().inspect_err(|_| { dev_err!(dev, "M3: the constructed InitData records do not match the G15 InitData structures\n") })?; - let cfg: &'static hw::HwConfig = match t6031 { - Some(e) => e.hwcfg(), - None => soc.hwcfg.ok_or(ENODEV)?, - }; + let cfg: &'static hw::HwConfig = soc.hwcfg.ok_or(ENODEV)?; let io_mappings = match experiment { Some(e) => e.io_mappings(), - None => match t6031 { - Some(e) => e.io_mappings(), - None => soc.io_mappings.ok_or(ENODEV)?, - }, + None => soc.io_mappings.ok_or(ENODEV)?, }; let runtime_hwdata_b = match experiment { Some(e) => e.hwdata_b(), - None => match t6031 { - Some(e) => e.hwdata_b(), - None => soc.hwdata_b.ok_or(ENODEV)?, - }, + None => soc.hwdata_b.ok_or(ENODEV)?, }; let mut pwr = hw::PwrConfig::load(dev, cfg).inspect_err(|e| { dev_err!(dev, "M3: cannot read the GPU power configuration from the device tree ({:?})\n", e) @@ -841,10 +804,7 @@ fn build_images( } // A SoC whose power model is still a stand-in caps the power target: every operating point's // power is scaled by the same factor, so the relative powers stay the boot loader's. - let power_cap_mw = match t6031 { - Some(e) => Some(e.power_cap_mw()), - None => soc.power_target_cap_mw, - }; + let power_cap_mw = soc.power_target_cap_mw; if let Some(cap) = power_cap_mw { let max = pwr.max_power_mw; if max > cap { @@ -863,7 +823,7 @@ fn build_images( let node = dev.of_node().ok_or(ENODEV)?; let dyncfg = hw::DynConfig { uat_ttb_base: firmware.resources.regions[0].base, - id: soc_identity(soc, cfg, t6031.and_then(|e| e.revision_id())), + id: soc_identity(soc, cfg, None), pwr, firmware_version: node .get_property::>(c_str!("apple,firmware-version")) @@ -877,7 +837,7 @@ fn build_images( split_pstates: true, // The cap is applied to the uploaded objects, for either source (`publish_pstates`). cap: None, - timestamp_base: Some(crate::agx_memory::TIMESTAMP_RANGE.start), + timestamp_base: Some(crate::m3::agx_memory::TIMESTAMP_RANGE.start), reference_ppm: true, runtime_hwdata_b: Some(runtime_hwdata_b), per_cluster_voltages: soc.per_cluster_voltages, @@ -922,12 +882,6 @@ fn build_images( write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; dev_info!(dev, "M3 G15G start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); - } else if let Some(e) = t6031 { - type B = raw::HwDataBG15V14_8_3; - let (a, b) = e.unit_masks(); - write(&mut hwdata, offset_of!(B, unit_mask_a), &a.to_le_bytes())?; - write(&mut hwdata, offset_of!(B, unit_mask_b), &b.to_le_bytes())?; - dev_info!(dev, "M3 G15C start: HwDataB unit masks +0x17c0 {:#x}, +0x17c8 {:#x}\n", a, b); } write(&mut hwdata, HWDATA_A, initdata::raw_bytes(&*c.hwdata_a))?; // The system counter value at InitData creation, the base of the firmware's first power and @@ -953,14 +907,6 @@ fn build_images( m.fast_die, m.alarm ); - } else if let Some(e) = t6031 { - let (fast, alarm) = e.mtr_masks(dev)?; - type A = raw::HwDataAG15V14_8_3; - for at in [offset_of!(A, fast_die0_sensor_mask), offset_of!(A, fast_die0_sensor_mask_2)] { - write(&mut hwdata, HWDATA_A + at, &fast.to_le_bytes())?; - } - write(&mut hwdata, HWDATA_A + offset_of!(A, fast_die0_sensor_mask_alt), &alarm.to_le_bytes())?; - dev_info!(dev, "M3: MTR sensor masks: fast-die {:#x}, alarm {:#x}\n", fast, alarm); } // The second leakage fuse value: the first entry of HwDataA's third leakage table (the first // table holds the core coefficient, from the builder). @@ -969,23 +915,15 @@ fn build_images( let at = HWDATA_A + offset_of!(A, cluster_tables) + 2 * size_of::<[u32; 8]>(); write(&mut hwdata, at, &second.to_bits().to_le_bytes())?; } - // The SoC's own values for words the shared builder writes otherwise. An armed T6031 copies - // the T8122 words only when asahi.t6031_fw_words=t8122. - let (hwdata_words, globals_words): (&[(usize, u32)], &[(usize, u32)]) = match t6031 { - Some(e) if e.fw_words() => ( - &crate::m3_soc::T8122_HWDATA_WORDS, - &crate::m3_soc::T8122_GLOBALS_WORDS, - ), - Some(_) => (&[], &[]), - None => (soc.hwdata_words, soc.globals_words), - }; + // The SoC's own values for words the shared builder writes otherwise. + let (hwdata_words, globals_words) = (soc.hwdata_words, soc.globals_words); for &(at, value) in hwdata_words { write(&mut hwdata, at, &value.to_le_bytes())?; } // T8122 with the G15 14.8.3 firmware (the only image the runtime accepts there): the 20 - // words of crate::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. - if crate::t8122_start::is_t8122(soc) - && core::ptr::eq(firmware.layout, &crate::m3_firmware::T8122_LAYOUT) + // words of crate::m3::t8122_hwdata at HwDataA +0x418c, unless asahi.t8122_hwdata_418c=0. + if crate::m3::t8122_start::is_t8122(soc) + && core::ptr::eq(firmware.layout, &crate::m3::m3_firmware::T8122_LAYOUT) && cfg.gpu_gen == hw::GpuGen::G15 { if m3_params::t8122_hwdata_418c() { @@ -1006,7 +944,7 @@ fn build_images( } // On T8122, log and require the two HwDataB words the firmware's power management depends // on: it powers the GPU cores up for a job only when +0xa38 and +0xa40 are both nonzero. - if core::ptr::eq(soc, &crate::m3_soc::T8122) { + if core::ptr::eq(soc, &crate::m3::m3_soc::T8122) { type B = raw::HwDataBG15V14_8_3; let a38 = read_u32(&hwdata, offset_of!(B, unk_460))?; let a40 = read_u32(&hwdata, offset_of!(B, unk_468))?; @@ -1019,7 +957,7 @@ fn build_images( return Err(EINVAL); } } - fill_io_mappings(dev, cfg, experiment, t6031, io_mappings, iomaps, &mut hwdata)?; + fill_io_mappings(dev, cfg, experiment, io_mappings, iomaps, &mut hwdata)?; images[HWDATA] = Some(hwdata); let mut globals = zeroed(GLOBALS)?; @@ -1047,7 +985,7 @@ fn build_images( if given { " (asahi.m3_ut_engagement)" } else { "" } ); } - if crate::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) + if crate::m3::m3_globals_debug::apply(soc.chip_id,m3_params::globals_debug_off(),&mut globals) .map_err(|_|EINVAL)? { dev_info!(dev,"M3: T6030 Globals debug word cleared at +0x2c\n"); } @@ -1237,7 +1175,7 @@ pub(crate) fn publish_runtime_cap(globals: &mut Buffer, cap: u32) -> Result { for off in GLOBALS_RUNTIME_CAP { globals.u32(off, 100 * cap)?; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); Ok(()) } diff --git a/drivers/gpu/drm/asahi/m3/m3_board.rs b/drivers/gpu/drm/asahi/m3/m3_board.rs index 5ecfd820b4e4f1..c44b2a42d99d55 100644 --- a/drivers/gpu/drm/asahi/m3/m3_board.rs +++ b/drivers/gpu/drm/asahi/m3/m3_board.rs @@ -1,16 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Board facts for the M3 runtime, read from the hardware and the device tree instead of being -//! fixed to one board. The per-SoC facts are in [`crate::m3_soc`]. +//! fixed to one board. The per-SoC facts are in [`crate::m3::m3_soc`]. use kernel::{bindings, c_str, device, device::Core, io::resource::Resource, of, platform, prelude::*, uapi}; -use crate::m3_resources::{ +use crate::m3::m3_resources::{ reserved_resource, Region, Resources, // }; -use crate::m3_soc::Soc; +use crate::m3::m3_soc::Soc; /// Whether the root node's compatible list contains `compatible`. fn board_is(compatible: &[u8]) -> bool { @@ -60,7 +60,7 @@ pub(crate) fn core_masks(soc: &Soc, mask: u64) -> [u32; uapi::DRM_ASAHI_MAX_CLUS /// The highest GPU frequency in the device tree OPP table, in kHz, or None when the GPU node /// has no populated OPP table. -pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { +pub(crate) fn max_frequency_khz(pdev: &platform::Device>) -> Option { let node = pdev.as_ref().of_node()?; let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; let mut max_hz: u64 = 0; @@ -77,7 +77,7 @@ pub(crate) fn max_frequency_khz(pdev: &platform::Device) -> Option { /// Every cluster of a point must have the same voltage, except on a SoC with per-cluster /// voltages, where the highest `opp-microvolt` cell orders the point. None without a readable /// OPP table. -pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Option<(u32, u32)> { +pub(crate) fn opp_table_shape(pdev: &platform::Device>, soc: &Soc) -> Option<(u32, u32)> { let node = pdev.as_ref().of_node()?; let opps = node.parse_phandle(c_str!("operating-points-v2"), 0)?; let mut points = KVec::new(); @@ -99,7 +99,7 @@ pub(crate) fn opp_table_shape(pdev: &platform::Device, soc: &Soc) -> Optio }; points.push((hz, uv), GFP_KERNEL).ok()?; } - crate::t8122_admission::opp_table_shape(points) + crate::m3::t8122_admission::opp_table_shape(points) } /// Register windows the runtime maps on `soc`: name, base, minimum size. The runtime uses the @@ -340,7 +340,7 @@ pub(crate) fn static_region(node: &of::Node, name: &CStr) -> Option, soc: &Soc) -> Result { +pub(crate) fn admit(pdev: &platform::Device>, soc: &Soc) -> Result { let dev = pdev.as_ref(); let refuse = |what: &str, err: Error| { dev_info!(dev, "M3: not admitted: {}\n", what); @@ -460,14 +460,14 @@ pub(crate) fn admit(pdev: &platform::Device, soc: &Soc) -> Result) -> bool { +pub(crate) fn has_pmp_link(pdev: &platform::Device>) -> bool { pdev.as_ref() .of_node() .is_some_and(|node| node.get_property::(c_str!("apple,pmp")).is_ok()) } -pub(crate) use crate::m3_firmware::{KnownImage, KNOWN_IMAGES, KNOWN_IMAGES_T8122}; -use crate::m3_firmware::image_info; +pub(crate) use crate::m3::m3_firmware::{KnownImage, KNOWN_IMAGES_T8122}; +use crate::m3::m3_firmware::image_info; fn sha256(data: &[u8]) -> [u8; 32] { let mut digest = [0u8; 32]; @@ -551,9 +551,8 @@ pub(crate) fn region_is_nomap(node: &of::Node, name: &CStr) -> bool { /// board, a render node there would let a stock userspace driver submit work the board may not /// run correctly. `auto` (-1) registers it only on the boards the runtime was validated on. pub(crate) fn expose_render_node(soc: &Soc) -> bool { - match *crate::module_parameters::m3_expose.value() { - 0 => false, - v if v > 0 => true, - _ => runtime_validated_board(soc), + match crate::m3::m3_params::expose() { + Some(expose) => expose, + None => runtime_validated_board(soc), } } diff --git a/drivers/gpu/drm/asahi/m3/m3_client.rs b/drivers/gpu/drm/asahi/m3/m3_client.rs index 6661222530a38c..8fc7791e939ba1 100644 --- a/drivers/gpu/drm/asahi/m3/m3_client.rs +++ b/drivers/gpu/drm/asahi/m3/m3_client.rs @@ -13,7 +13,6 @@ use core::sync::atomic::{ AtomicU32, - AtomicU64, Ordering, // }; @@ -23,8 +22,7 @@ use kernel::{ }; use crate::{ - driver::AsahiDevice, - agx_uapi::{ + m3::agx_uapi::{ GpuAccess, GpuAddressSpace, // }, @@ -38,28 +36,14 @@ const AGX3_PARAMS_SIZE: u64 = core::mem::size_of:: Self { - Self { - params_size: AtomicU64::new(0), - } - } - - /// Record a successful GET_PARAMS for the global parameters. - pub(crate) fn note_params(&self, size: u64) { - self.params_size.fetch_max(size, Ordering::Relaxed); - } - - /// Whether this file may create a VM. Only G15 devices are gated. - pub(crate) fn admit_vm(&self, device: &AsahiDevice, flags: u32) -> Result { - let size = self.params_size.load(Ordering::Relaxed); - if crate::g16_profile::client_admitted(device.gpu()?.params()?.gpu_hal_generation, flags, size >= AGX3_PARAMS_SIZE) { +/// Whether a file may create a VM on the M3 runtime: it must have asked GET_PARAMS for the +/// whole global parameter structure (so it knows this GPU runs AGX3 command streams), and pass +/// VM_CREATE flags 0 (legacy HAL). `size` is the largest GET_PARAMS size the file asked for. +pub(crate) fn admit_vm(device: &kernel::device::Device, size: u64, flags: u32) -> Result { + { + // The positive acknowledgement of the command ABI (7.1.12 `g16_profile::client_admitted`): + // GET_PARAMS coverage, and VM_CREATE flags 0 on the legacy-HAL M3 runtime. + if size >= AGX3_PARAMS_SIZE && flags == 0 { return Ok(()); } let refusals = REFUSALS.fetch_add(1, Ordering::Relaxed).saturating_add(1); @@ -74,7 +58,7 @@ impl ClientGate { "" }; dev_info!( - device.as_ref(), + device, "M3: refusing a VM for {}[{}]: it asked GET_PARAMS for {} of the {} parameter bytes, so it cannot know this GPU runs AGX3 command streams; M3-aware userspace is needed{}\n", name, task.group_leader().pid(), @@ -200,7 +184,9 @@ pub(crate) fn check_compute_launches( /// `Vm::read_mapped_bytes` holds the VM execution lock from translation until the copy, so the /// pages cannot be unmapped and freed while they are cleaned and read. fn read_through_cache(vm: &mmu::Vm, iova: u64, out: &mut [u8]) -> Result { - vm.read_mapped_bytes(iova, out, |src, len| clean_invalidate(src as usize, len)) + // 7.2's `Vm::read_user` cleans and invalidates each line to the point of coherency before + // the copy, under the VM execution lock. + vm.read_user(iova, out) } /// Clean and invalidate the data cache lines covering `start..start + len` to the point of diff --git a/drivers/gpu/drm/asahi/m3/m3_completion.rs b/drivers/gpu/drm/asahi/m3/m3_completion.rs index 08114c4d8cc8ad..e06fcded9047ef 100644 --- a/drivers/gpu/drm/asahi/m3/m3_completion.rs +++ b/drivers/gpu/drm/asahi/m3/m3_completion.rs @@ -3,7 +3,7 @@ //! Completion work owned only by the M3 runtime. use kernel::{prelude::*, sync::Arc, workqueue::{self, impl_has_work, new_work, Work, WorkItem}}; -use crate::{driver, m3_drm::Shared}; +use crate::m3::{m3_drm::Shared, m3_rtkit::State}; #[pin_data] pub(crate) struct Completion { @@ -23,7 +23,7 @@ impl_has_work! { impl HasWork for Completion { self.work } } impl WorkItem for Completion { type Pointer = Arc; fn run(owner: Arc) { - use crate::m3_runtime::Service; + use crate::m3::m3_runtime::Service; // Snapshots after a notification before sleeping for the next one: the completion // stamps can trail the firmware's event message. const TRAILING_POLLS: u32 = 4; @@ -59,8 +59,8 @@ impl WorkItem for Completion { } } } -pub(crate) fn queue(dev: &driver::AsahiDevice) { - if let Some(work) = dev.completion.as_ref() { +pub(crate) fn queue(state: &State) { + if let Some(work) = state.completion.as_ref() { let _ = workqueue::system_highpri().enqueue(work.clone()); } } diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index e7469d31b496fe..eab8bf71f7bcdf 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -1,15 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Owned RTKit 2419 M3 compute graph, populated with a validated user CDM. use kernel::prelude::*; -use crate::m3_queue_layout as queue; -use crate::m3_state_layout as state; -use crate::{m3_compute_layout as command, m3_compute_sequence as microsequence, - m3_pool_layout::GpuRegion, m3_init_layout::Region}; -use crate::{driver,mmu,agx_memory::{self},m3_memory::Buffer,m3_compute_storage as storage}; -use crate::m3_timeline as timeline; +use crate::m3::m3_queue_layout as queue; +use crate::m3::m3_state_layout as state; +use crate::{m3::m3_compute_layout as command, m3::m3_compute_sequence as microsequence, + m3::m3_pool_layout::GpuRegion, m3::m3_init_layout::Region}; +use crate::{driver,mmu,m3::agx_memory::{self},m3::m3_memory::Buffer,m3::m3_compute_storage as storage}; +use crate::m3::m3_timeline as timeline; use storage::{NOTIFIER,PREEMPTION,GPU_CONTEXT,JOB_LIST,QUEUE_SCRATCH,TIMESTAMPS, TAIL_SCRATCH,MICROSEQUENCE_SCRATCH,Mapping}; -const UMA_POOL:crate::m3_pool::Owners=crate::m3_pool::Owners { +const UMA_POOL:crate::m3::m3_pool::Owners=crate::m3::m3_pool::Owners { backing:storage::POOL_BACKING,pages:storage::POOL_PAGES,table:storage::POOL_TABLE, manager:storage::POOL,counter:storage::COUNTER,table_index:2, }; @@ -40,11 +40,14 @@ pub(crate) struct Compute { barriers: Option, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). registers: command::RegisterSet, + /// Last: drops after the command aliases. + _lower_tlbi: crate::m3::m3_memory::LowerTlbi, } impl Compute { - pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, vm: &mmu::Vm, - stats: Region, control: crate::agx_compute::Control, + pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, vm: &mmu::Vm, + stats: Region, control: crate::m3::agx_compute::Control, registers: command::RegisterSet) -> Result { + let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new(); let mut client_command=None; @@ -101,10 +104,10 @@ impl Compute { // the invalidation. crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; - Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers}) + Ok(Self {_client_command:client_command.ok_or(EINVAL)?,objects,command_bytes,sequence_bytes,_binding:binding,stats,last_progress_ns:0,sequence:1,head:1,cached_views:KVec::new(),slot:0,batch_count:1,checkpoint:(0,0,0),barriers:None,registers,_lower_tlbi:lower_tlbi}) } fn encode_command(objects: &mut [Buffer], bytes: &mut [u8], context: u32, - control: crate::agx_compute::Control, sequence: u64, slot: usize, + control: crate::m3::agx_compute::Control, sequence: u64, slot: usize, register_set: command::RegisterSet) -> Result { let offset = |owner:usize| storage::slot_offset(owner,slot).map_err(|_|EINVAL); let fw = |owner: usize| queue::FirmwareVa::new(objects[owner].va()+offset(owner)? as u64).map_err(|_| EINVAL); @@ -164,7 +167,7 @@ impl Compute { objects[PREEMPTION].fill(0xcc) } pub(crate) fn replay(&mut self, uat: &mmu::Uat, vm: &mmu::Vm, - control: crate::agx_compute::Control) -> Result { + control: crate::m3::agx_compute::Control) -> Result { if !self.complete()? { return Err(EBUSY); } // All prior flushed stamps/events and engine idle were checked by // Runtime before entry. Keep pool/queue ownership, remap only client @@ -211,7 +214,7 @@ impl Compute { /// Append only while the previous batch is fully retired and this one /// remains unpublished. Firmware serializes each command, including its /// original completion flush, in this one compute queue. - pub(crate) fn append(&mut self,control:crate::agx_compute::Control)->Result { + pub(crate) fn append(&mut self,control:crate::m3::agx_compute::Control)->Result { if self.batch_count>=storage::SLOTS {return Err(E2BIG);} self.slot=self.batch_count; for i in storage::RESET { @@ -268,20 +271,20 @@ impl Compute { } agx_memory::publish();Ok(()) } - pub(crate) fn set_attachments(&mut self,attachments:&crate::agx_attachments::Attachments)->Result { + pub(crate) fn set_attachments(&mut self,attachments:&crate::m3::agx_attachments::Attachments)->Result { let sequence=self.offset(storage::MICROSEQUENCE)?; self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::ATTACHMENTS,&attachments.0)?; self.objects[storage::MICROSEQUENCE].write(sequence+microsequence::HAS_ATTACHMENTS,&[u8::from(attachments.count()!=0)])?; agx_memory::publish();Ok(()) } fn stamp(&self) -> u32 { timeline::stamp(0xc1000000,self.sequence) } - pub(crate) fn completion_dependency(&self)->Result { - Ok(crate::m3_sync_layout::CompletionDependency { + pub(crate) fn completion_dependency(&self)->Result { + Ok(crate::m3::m3_sync_layout::CompletionDependency { stamp:queue::FirmwareVa::new(self.objects[storage::FW_STAMP].va()).map_err(|_|EINVAL)?, value:self.stamp(),event:2, }) } - pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat)->Result { + pub(crate) fn reserve_barriers(&mut self,dev:&driver::AsahiDrmDevice,uat:&mmu::Uat)->Result { if self.barriers.is_none() { self.barriers=Some(Buffer::at(dev,uat.kernel_vm(),None,None,storage::SLOTS*64,false)?); } @@ -290,17 +293,17 @@ impl Compute { /// Insert a dependency before the last appended, still unpublished CDM. /// Its command ordinal and timestamp slot are unchanged; only the ring /// gains an entry. The whole batch must retire before this storage is reused. - pub(crate) fn insert_dependency(&mut self,dependency:crate::m3_sync_layout::CompletionDependency)->Result { + pub(crate) fn insert_dependency(&mut self,dependency:crate::m3::m3_sync_layout::CompletionDependency)->Result { if self.batch_count==0 || self.slot>=storage::SLOTS || dependency.event!=1 { return Err(EINVAL); } let value=self.stamp(); let barriers=self.barriers.as_mut().ok_or(EINVAL)?; let offset=self.slot*64; - let mut bytes=[0;crate::m3_sync_layout::BARRIER_SIZE]; - crate::m3_sync_layout::Barrier { + let mut bytes=[0;crate::m3::m3_sync_layout::BARRIER_SIZE]; + crate::m3::m3_sync_layout::Barrier { stamp:dependency.stamp,wait_value:dependency.value,event:dependency.event, - self_value:value,uuid:0xc10000,dependency:crate::m3_sync_layout::Dependency::PreviousPass, + self_value:value,uuid:0xc10000,dependency:crate::m3::m3_sync_layout::Dependency::PreviousPass, }.encode(&mut bytes).map_err(|_|EINVAL)?; barriers.write(offset,&bytes)?; let address=barriers.va()+offset as u64; @@ -333,7 +336,7 @@ impl Compute { pub(crate) fn head(&self) -> u16 { self.head } pub(crate) fn ordinal(&self)->u64 {self.sequence} pub(crate) fn first(&self) -> bool { self.sequence == self.batch_count as u64 } - pub(crate) fn progress(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn progress(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Report real completions by elapsed time as well as sequence count; // batches need not land on a multiple of 128. if !crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { @@ -360,7 +363,7 @@ impl Compute { && self.objects[storage::QUEUE_STATE].read_u32(queue::DONE)?==u32::from(self.head) && self.objects[storage::QUEUE_STATE].read_u32(queue::READ)?==u32::from(self.head)) } - pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { for off in (0..512).step_by(128) { let mut raw=[0u8;128]; self.objects[storage::COMMAND].read(off,&mut raw)?; dev_info!(dev.as_ref(),"M3 post-command {:#x} {:02x?}\n",off,raw); diff --git a/drivers/gpu/drm/asahi/m3/m3_config.rs b/drivers/gpu/drm/asahi/m3/m3_config.rs index 59277aaefb6817..c4a44c4b4f3b0e 100644 --- a/drivers/gpu/drm/asahi/m3/m3_config.rs +++ b/drivers/gpu/drm/asahi/m3/m3_config.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Owned RTKit 2419.140.12 configuration with named initialization references. use kernel::prelude::*; -use crate::m3_init_layout as init; -use crate::m3_init_storage as storage; +use crate::m3::m3_init_layout as init; +use crate::m3::m3_init_storage as storage; use storage::{REGION_A,RUNTIME_POINTERS,HARDWARE_DATA,UNKNOWN_PAIR,FWLOG_PAYLOAD, UNKNOWN_C0,UNKNOWN_C1,UNKNOWN_C3,GLOBALS,GLOBALS_POWER,CONTROL_REGION,RUNTIME_FLAGS, FW_CONTROL_STATE,FW_CONTROL_RING}; -use crate::{driver,agx_memory::{self},m3_memory::Buffer,mmu,pgtable::prot}; +use crate::{driver,m3::agx_memory::{self},m3::m3_memory::Buffer,mmu,pgtable::prot}; kernel::static_assert!(core::mem::size_of::() == init::fwlog::ENTRY_SIZE); kernel::static_assert!(core::mem::size_of::() == init::fwlog::PAYLOAD_SIZE); kernel::static_assert!(::SUB_CHANNELS == init::fwlog::CHANNELS); @@ -49,12 +49,10 @@ impl Records { pub(crate) struct Config { objects: KVec, - #[cfg(CONFIG_DEV_COREDUMP)] - fault_reserve: Option, _iomaps: KVec, pub(crate) completed_events: u64, - pstates: crate::m3_adt_config::PstateWatch, - thermal: crate::m3_thermal::Governor, + pstates: crate::m3::m3_adt_config::PstateWatch, + thermal: crate::m3::m3_thermal::Governor, request_top: u32, request_value: u32, request_checked: bool, @@ -63,47 +61,9 @@ pub(crate) struct Config { request_observed: u32, } impl Config { - /// Snapshot owned RAM only. Firmware can still update these buffers; - /// per-record time intervals explicitly describe sequential observations. - #[cfg(CONFIG_DEV_COREDUMP)] - pub(crate) fn fault_snapshot(&mut self, dev: &driver::AsahiDevice, primary: Error, gpu_pending: bool) -> Result { - let mut dump = self.fault_reserve.take().ok_or(ENOMEM)?; - dump.begin(0); - let mut cause = [0u8; 16]; - cause[..4].copy_from_slice(&1u32.to_le_bytes()); - cause[4..8].copy_from_slice(&primary.to_errno().to_le_bytes()); - cause[8..16].copy_from_slice(&u64::from(gpu_pending).to_le_bytes()); - dump.record("host-first-error", 0, cause.len(), |out| { - out.copy_from_slice(&cause); Ok(()) - })?; - for (name, index) in [ - ("root", storage::ROOT), ("runtime", RUNTIME_POINTERS), - ("hardware", HARDWARE_DATA), ("globals", GLOBALS), - ("control", CONTROL_REGION), ("flags", RUNTIME_FLAGS), - // The power-controller block, and the host-to-firmware device-control and - // firmware-control queues: their state words show whether the firmware read a - // queued control (read index) and what it was sent. - ("power", GLOBALS_POWER), - ("devctl-state", storage::DEVICE_CONTROL * 2), - ("devctl-ring", storage::DEVICE_CONTROL * 2 + 1), - ("fwctl-state", storage::FW_CONTROL_STATE), - ("fwctl-ring", storage::FW_CONTROL_RING), - ("event-state", storage::EVENT * 2), - ("event-ring", storage::EVENT * 2 + 1), - ("fwlog-state", storage::FIRMWARE_LOG * 2), - ("fwlog-ring", storage::FIRMWARE_LOG * 2 + 1), - ("fwlog-payload", FWLOG_PAYLOAD), - ("trace-state", storage::TRACE * 2), - ("trace-ring", storage::TRACE * 2 + 1), - ] { - let buffer = &mut self.objects[index]; - dump.record(name, buffer.va(), buffer.size(), |out| buffer.read(0, out))?; - } - dump.publish(dev.as_ref()) - } - pub(crate) fn new(dev: &driver::AsahiDevice, uat: &mmu::Uat, firmware: &crate::m3_firmware::Firmware, - contents: &crate::m3_adt_config::Contents) -> Result { + pub(crate) fn new(dev: &driver::AsahiDrmDevice, uat: &mmu::Uat, firmware: &crate::m3::m3_firmware::Firmware, + contents: &crate::m3::m3_adt_config::Contents) -> Result { contents.check_for_upload()?; let mut objects = KVec::new(); let mut iomaps = KVec::new(); @@ -125,8 +85,8 @@ impl Config { for io in contents.iomaps { // Read-write unless the contents say otherwise (only the T8122 start experiment). let protection=if contents.read_only_slots==0 {prot::PROT_FW_MMIO_RW} - else {crate::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; - let map=uat.kernel_vm().map_io(io.address,io.physical.try_into()?,io.size,protection)?; + else {crate::m3::t8122_start::mmio_prot(contents.read_only_slots,io.slot)}; + let map=uat.kernel_vm().map_io_m3(io.address,io.physical.try_into()?,io.size,protection)?; let owned=init::Region::new(map.iova(),io.size).map_err(|_|EINVAL)?; let field=io.pointer_field().map_err(|_|EINVAL)?; objects[HARDWARE_DATA].u64(field,io.pointer(owned).map_err(|_|EINVAL)?)?; @@ -139,7 +99,7 @@ impl Config { // The root record carries the G15 14.8.3 InitData version, which is every T6030 // image's. Only the T8122 start experiment gives the firmware another one // (`asahi.t8122_initdata_version`). - if firmware.initdata_magic!=crate::m3_firmware::G15_V14_8_3_INITDATA { + if firmware.initdata_magic!=crate::m3::m3_firmware::G15_V14_8_3_INITDATA { objects[storage::ROOT].u64(0,firmware.initdata_magic)?; } // Vertex/tessellation loops need not retire a primitive at every @@ -171,26 +131,17 @@ impl Config { return Err(EINVAL); } let (hwdata, globals) = objects.split_at_mut(GLOBALS); - crate::m3_adt_config::publish_pstates(dev.as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; + crate::m3::m3_adt_config::publish_pstates(dev.as_ref().as_ref(), &mut hwdata[HARDWARE_DATA], &mut globals[0], &contents.pstates)?; // Firmware ktrace is for explicit detailed diagnostics. Normal games // still drain error/events, but need not generate trace records for // every firmware operation on the command-processing core. objects[RUNTIME_FLAGS].u32(0,u32::from(crate::debug::debug_enabled( crate::debug::DebugFlags::SubmitTiming)))?; - crate::m3_adt_config::check_upload(dev.as_ref(), firmware, contents.iomaps, &mut objects)?; + crate::m3::m3_adt_config::check_upload(dev.as_ref().as_ref(), firmware, contents.iomaps, &mut objects)?; agx_memory::publish(); - let thermal = crate::m3_thermal::Governor::new(dev.as_ref(), &contents.pstates)?; - #[cfg(CONFIG_DEV_COREDUMP)] - let fault_reserve = match crate::agx_fault::Dump::new_m3() { - Ok(dump) => Some(dump), - Err(error) => { - dev_warn!(dev.as_ref(), "M3: fault snapshot reserve unavailable: {:?}\n", error); - None - } - }; + let thermal = crate::m3::m3_thermal::Governor::new(dev.as_ref().as_ref(), &contents.pstates)?; Ok(Self { objects, _iomaps: iomaps, completed_events: 0, - #[cfg(CONFIG_DEV_COREDUMP)] fault_reserve, - pstates: crate::m3_adt_config::PstateWatch::new(contents.pstates), thermal, + pstates: crate::m3::m3_adt_config::PstateWatch::new(contents.pstates), thermal, request_top:contents.table.0,request_value:0,request_checked:false, request_check_at:None,request_writes:0,request_observed:0 }) } @@ -203,22 +154,22 @@ impl Config { objects[CONTROL_REGION].write(0, &records.control) } pub(crate) fn root(&self) -> u64 { self.objects[storage::ROOT].va() } - pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDevice)->Result { - let value=self.objects[GLOBALS].read_u32(crate::m3_globals_debug::OFFSET)?; + pub(crate) fn log_globals_debug(&mut self,dev:&driver::AsahiDrmDevice)->Result { + let value=self.objects[GLOBALS].read_u32(crate::m3::m3_globals_debug::OFFSET)?; dev_info!(dev.as_ref(),"M3_GLOBALS_DEBUG off={} value={} request_level={}\n", - u8::from(crate::m3_params::globals_debug_off()),value,crate::m3_params::consistent_perf()); + u8::from(crate::m3::m3_params::globals_debug_off()),value,crate::m3::m3_params::consistent_perf()); Ok(()) } pub(crate) fn performance_request(&mut self,chip:u32,busy:bool)->Result { - let Some(value)=crate::m3_consistent_perf::request(chip, - crate::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; + let Some(value)=crate::m3::m3_consistent_perf::request(chip, + crate::m3::m3_params::consistent_perf(),busy,self.request_top) else {return Ok(());}; if value==0 && self.request_value==0 {return Ok(());} let now=kernel::time::Instant::::now(); let elapsed=self.request_check_at.map(|last|(now-last).as_nanos()); - if !crate::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { + if !crate::m3::m3_consistent_perf::refresh_due(value,self.request_value,elapsed) { return Ok(()); } - type P=crate::fw::initdata::raw::PowerCtlBlock; + type P=crate::m3::g15_raw::raw::PowerCtlBlock; let at=core::mem::offset_of!(P,consistent_perf_state); if !self.request_checked { let mut flag=[0]; @@ -240,14 +191,14 @@ impl Config { /// Move the runtime cap by temperature (the thermal limit), then check the performance /// state the firmware reports against the ceiling and the runtime cap. Called after boot and /// after every job, by the job thread. - pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDevice, device: &crate::m3_device::Device, what: &str) -> Result { + pub(crate) fn check_pstate(&mut self, dev: &driver::AsahiDrmDevice, device: &crate::m3::m3_device::Device, what: &str) -> Result { let now = kernel::time::Instant::::now(); - if let Some(cap) = self.thermal.update(dev.as_ref(), now) { - crate::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; + if let Some(cap) = self.thermal.update(dev.as_ref().as_ref(), now) { + crate::m3::m3_adt_config::publish_runtime_cap(&mut self.objects[GLOBALS], cap)?; } let allowed = self.thermal.allowed(now); - let (over, peak) = self.pstates.check(dev.as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; - self.thermal.note_check(dev.as_ref(), now, over, peak)?; + let (over, peak) = self.pstates.check(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], || device.pstate_register(), what, allowed)?; + self.thermal.note_check(dev.as_ref().as_ref(), now, over, peak)?; self.thermal.busy(now); Ok(()) } @@ -256,14 +207,14 @@ impl Config { init::Region::new(owner.va(), owner.size()).map_err(|_| EINVAL) } - pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDevice, - device: &crate::m3_device::Device, + pub(crate) fn log_active_power(&mut self, dev: &driver::AsahiDrmDevice, + device: &crate::m3::m3_device::Device, sequence: u32) -> Result { let cap = self.thermal.allowed(kernel::time::Instant::::now()); - self.pstates.log_active(dev.as_ref(), &mut self.objects[HARDWARE_DATA], + self.pstates.log_active(dev.as_ref().as_ref(), &mut self.objects[HARDWARE_DATA], device.pstate_register()?, cap, sequence)?; - type P=crate::fw::initdata::raw::PowerCtlBlock; - type G=crate::fw::initdata::raw::GlobalsG15V14_8_3; + type P=crate::m3::g15_raw::raw::PowerCtlBlock; + type G=crate::m3::g15_raw::raw::GlobalsG15V14_8_3; let request=self.objects[GLOBALS_POWER].read_u32(core::mem::offset_of!(P,consistent_perf_state))?; let cap_word=self.objects[GLOBALS].read_u32(core::mem::offset_of!(G,perf_state_cap))?; let power_if=core::mem::offset_of!(G,power_if_target); @@ -272,13 +223,13 @@ impl Config { let mut override_active=[0]; self.objects[GLOBALS_POWER].read(core::mem::offset_of!(P,perf_ctrl_override_active),&mut override_active)?; dev_info!(dev.as_ref(),"M3_PERF_REQUEST sample={} level={} request={} cap_word={} target1={} target2={} writes={} observed={} override_active={}\n", - sequence,crate::m3_params::consistent_perf(),request,cap_word,target1,target2, + sequence,crate::m3::m3_params::consistent_perf(),request,cap_word,target1,target2, self.request_writes,self.request_observed,override_active[0]); Ok(()) } pub(crate) fn render_pb(&mut self) -> Result { - self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3_render::PB_PAGES).map_err(|_|EINVAL)?) + self.objects[UNKNOWN_C1].write(0,&storage::parameter_buffer(crate::m3::m3_render::PB_PAGES).map_err(|_|EINVAL)?) } pub(crate) fn submit_queue(&mut self, kind:usize, queue:u64, head:u16,event:u8,new:bool)->Result { if kind>2 {return Err(EINVAL);} @@ -316,7 +267,7 @@ impl Config { Ok(self.objects[RUNTIME_POINTERS].read_u32(0x314)? == 0xabcdabcd && self.objects[RUNTIME_FLAGS].read_u32(4)? == 1) } - pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn log_ready(&mut self, dev: &driver::AsahiDrmDevice) -> Result { let magic=self.objects[RUNTIME_POINTERS].read_u32(0x314)?; let state=self.objects[RUNTIME_POINTERS].read_u32(0x2fc)?; let a=self.objects[RUNTIME_FLAGS].read_u32(4)?; @@ -345,7 +296,7 @@ impl Config { /// at +0x4580/+0x4590/+0x45a0, the firmware's own recovery information /// at +0x40a0, and the details it leaves for a recovery event at +0x44a0. /// Nothing is acknowledged, cleared or resumed here. - pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDevice, events: u64) -> Result { + pub(crate) fn log_recovery_state(&mut self, dev: &driver::AsahiDrmDevice, events: u64) -> Result { let control=&mut self.objects[CONTROL_REGION]; let halt_count=control.read_u64(0x4580)?; let halted=control.read_u32(0x4590)?; @@ -360,7 +311,7 @@ impl Config { /// Read-only: log the event ring entries from the host's read index to the firmware's write /// index (at most 16), as `drain` reads them. Nothing is consumed or acknowledged, and the /// completion count is not changed. - pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn log_event_ring(&mut self, dev: &driver::AsahiDrmDevice) -> Result { const SIZE: usize = 0x38; const COUNT: usize = 256; let r = self.objects[storage::EVENT*2].read_u32(0)? as usize; @@ -395,7 +346,7 @@ impl Config { pub(crate) fn control_done(&mut self, next: u32) -> Result { Ok(self.objects[storage::DEVICE_CONTROL*2].read_u32(0)? == next && self.objects[storage::DEVICE_CONTROL*2].read_u32(0x10)? == next) } - pub(crate) fn drain(&mut self, dev: &driver::AsahiDevice) -> Result { + pub(crate) fn drain(&mut self, dev: &driver::AsahiDrmDevice) -> Result { // Event, trace, stats and six independent firmware log rings. for (state, ring, size, count, channels) in [(storage::EVENT*2,storage::EVENT*2+1,0x38,256,1), (storage::TRACE*2,storage::TRACE*2+1,0x38,512,1), diff --git a/drivers/gpu/drm/asahi/m3/m3_device.rs b/drivers/gpu/drm/asahi/m3/m3_device.rs index 1c2f0b1e8866da..c43017a8933c0f 100644 --- a/drivers/gpu/drm/asahi/m3/m3_device.rs +++ b/drivers/gpu/drm/asahi/m3/m3_device.rs @@ -12,6 +12,7 @@ use kernel::{ io::{ mem::IoMem, Io, + IoBase, }, platform, prelude::*, @@ -19,16 +20,16 @@ use kernel::{ time::{delay::fsleep, Delta, Instant, Monotonic}, }; -use crate::m3_firmware::Firmware; -use crate::m3_soc::Soc; +use crate::m3::m3_firmware::Firmware; +use crate::m3::m3_soc::Soc; const ASC_CPU_CONTROL: usize = 0x44; const ASC_CPU_RUN: u32 = 1 << 4; pub(crate) struct Device { dev: ARef, - asc: Pin>>>, - sgx: Pin>>, + asc: Devres>, + sgx: Devres>, firmware: Firmware, core_mask: u64, soc: &'static Soc, @@ -37,34 +38,28 @@ pub(crate) struct Device { impl Device { /// `experiment` is the T8122 start experiment's values, on an armed T8122 only: it decides - /// the SGX setup write, which the T8122 table lacks. `t6031` is the same for an armed T6031. - pub(crate) fn new(pdev: &platform::Device, firmware: Firmware, soc: &'static Soc, - experiment: Option<&crate::t8122_start::Experiment>, - t6031: Option<&crate::t6031_start::Experiment>) -> Result { + /// the SGX setup write, which the T8122 table lacks. + pub(crate) fn new(pdev: &platform::Device>, firmware: Firmware, soc: &'static Soc, + experiment: Option<&crate::m3::t8122_start::Experiment>) -> Result { // Map ASC before taking a vote. The larger SGX aperture contains // this control window and the separate mailbox provider, so it is // mapped without a conflicting claim over those child resources. // Only the 16 KiB control window is used. Some device trees describe // the ASC block as a larger range that also covers the mailbox, which // the mailbox driver has already claimed, so do not claim the range. - let asc = KBox::pin_init( - pdev.io_request_by_name(c_str!("asc")) - .ok_or(EINVAL)? - .iomap_sized::<0x4000>(), - GFP_KERNEL, - )?; + let asc = pdev.io_request_by_name(c_str!("asc")) + .ok_or(EINVAL)? + .iomap_sized::<0x4000>()? + .into_devres()?; dev_info!(pdev.as_ref(), "M3: ASC aperture mapped\n"); - let sgx = KBox::pin_init( - pdev.io_request_by_name(c_str!("sgx")) - .ok_or(EINVAL)? - .iomap(), - GFP_KERNEL, - )?; + let sgx = pdev.io_request_by_name(c_str!("sgx")) + .ok_or(EINVAL)? + .iomap()? + .into_devres()?; dev_info!(pdev.as_ref(), "M3: SGX aperture mapped\n"); - let pmp = crate::m3_board::has_pmp_link(pdev); - let t8122 = crate::t8122_start::is_t8122(soc); - let t6031_soc = crate::t6031_start::is_t6031(soc); - if pmp && !t8122 && !t6031_soc { + let pmp = crate::m3::m3_board::has_pmp_link(pdev); + let t8122 = crate::m3::t8122_start::is_t8122(soc); + if pmp && !t8122 { dev_err!(pdev.as_ref(), "M3: optional apple,pmp GPU link is unsupported\n"); return Err(ENOTSUPP); } @@ -78,7 +73,7 @@ impl Device { } else { dev_info!(pdev.as_ref(), "M3: no apple,pmp link; GPU power is left to its power domain\n"); } - if asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + if asc.access(pdev.as_ref())?.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { dev_err!( pdev.as_ref(), "M3 {}: ASC already running; refusing to take ownership\n", @@ -103,19 +98,15 @@ impl Device { // failed acquisition releases it only after the ASC-stopped check in Drop. device.power_vote = true; to_result(unsafe { bindings::apple_pmp_set_device_power(0x0f, 5, 1) })?; - if t6031_soc { - dev_info!(pdev.as_ref(), "M3 G15C: PMP GPU device 5 power acknowledged\n"); - } else { - dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); - } + dev_info!(pdev.as_ref(), "M3 G15G: PMP GPU device 5 power acknowledged\n"); } - let registers = device.sgx.access(pdev.as_ref())?; + let registers = device.sgx.access(pdev.as_ref())?.as_view(); let version = registers.try_read32(0xd04000)?; let counts = registers.try_read32(0xd04010)?; // The core-enable words: the second only on a SoC with more than 32 core slots. let mut core_mask = u64::from(registers.try_read32(0xe01500)?); - if crate::m3_board::core_slots(soc) > 32 { + if crate::m3::m3_board::core_slots(soc) > 32 { core_mask |= u64::from(registers.try_read32(0xe01504)?) << 32; } dev_info!(pdev.as_ref(), "M3 {}: power acknowledged, version={:#010x} counts={:#010x} core-mask={:#x}, firmware={}\n", @@ -124,7 +115,7 @@ impl Device { // fused core mask varies with SKU; absent cores must never be enabled. if version & soc.id.version_mask != soc.id.version || counts & soc.id.counts_mask != soc.id.counts - || !crate::m3_board::core_mask_valid(soc, core_mask) + || !crate::m3::m3_board::core_mask_valid(soc, core_mask) { return Err(ENODEV); } @@ -138,21 +129,7 @@ impl Device { } e.sgx_setup() } - None => match t6031 { - Some(e) => { - match e.sgx_setup() { - Some((offset, value)) => dev_warn!( - pdev.as_ref(), - "M3 G15C start: SGX setup write: SGX+{:#x} = {:#x}\n", - offset, - value - ), - None => dev_info!(pdev.as_ref(), "M3 G15C start: no SGX setup write\n"), - } - e.sgx_setup() - } - None => Some(soc.sgx_setup.ok_or(ENODEV)?), - }, + None => Some(soc.sgx_setup.ok_or(ENODEV)?), }; if let Some((setup_offset, setup_value)) = setup { registers.try_write32(setup_value, setup_offset)?; @@ -173,7 +150,8 @@ impl Device { /// accesses selectable (`asahi.m3_retire_mmio`); with neither it touches no GPU register. pub(crate) fn check_idle_parts(&self,busy:bool,faults:bool)->Result { if !busy && !faults {return Ok(());} - let sgx=self.sgx.try_access().ok_or(ENODEV)?; + let sgx_guard=self.sgx.try_access().ok_or(ENODEV)?; + let sgx=sgx_guard.as_view(); if busy && (sgx.try_read64(0xc020)? | sgx.try_read64(0xc120)?) & 1 != 0 { return Err(EBUSY); } @@ -190,34 +168,16 @@ impl Device { if fault!=0 {return Err(EIO);} Ok(()) } - pub(crate) fn check_drm(&self, pdev: &platform::Device) -> Result { - use kernel::dma::{Device as _, DmaMask}; - // SAFETY: This is the admitted M3 GPU's 42-bit physical DMA capability. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(42)?)? }; - crate::mmu::check_handoff_guard()?; - dev_info!(pdev.as_ref(), "M3 {}: GPU handoff lock checked.\n", self.soc.gpu_name); - for _ in 0..2 { - let drm: ARef = kernel::drm::Device::new( - pdev.as_ref(), crate::driver::AsahiData::new(pdev, None, true))?; - if drm.gpu().is_ok() { return Err(EIO); } - let mut object = crate::gem::new_kernel_object(&drm, 0x4000)?; - object.vmap()?.memset(0); - drop(object); - drop(drm); - } - dev_info!(pdev.as_ref(), "M3 {}: DRM data initialized; backend absent, device unregistered\n", self.soc.gpu_name); - Ok(()) - } - /// The raw performance-state register (SGX+0xe01000). pub(crate) fn pstate_register(&self) -> Result { - self.sgx.try_access().ok_or(ENODEV)?.try_read32(0xe01000) + self.sgx.try_access().ok_or(ENODEV)?.as_view().try_read32(0xe01000) } /// Read-only snapshot. Do not select or acknowledge fault banks here; /// the firmware owns those registers until recovery is ported. pub(crate) fn log_engine_state(&self, vm: &crate::mmu::Vm) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let sgx_guard = self.sgx.try_access().ok_or(ENODEV)?; + let sgx = sgx_guard.as_view(); let pstate = sgx.try_read32(0xe01000)? & 0xf; if pstate == 0 { dev_info!(self.dev.as_ref(), "M3 {}: GPU is powered down; skipping engine/MMU register snapshot\n", self.soc.gpu_name); @@ -240,10 +200,10 @@ impl Device { let service=sgx.try_read32(0xa010)?; let debug=sgx.try_read32(0xa000)?; let mut cores=u64::from(sgx.try_read32(0xe01500)?); - if crate::m3_board::core_slots(self.soc) > 32 { + if crate::m3::m3_board::core_slots(self.soc) > 32 { cores |= u64::from(sgx.try_read32(0xe01504)?) << 32; } - for core in 0..crate::m3_board::core_slots(self.soc).min(64) { + for core in 0..crate::m3::m3_board::core_slots(self.soc).min(64) { if cores & (1u64< &Firmware { &self.firmware } - pub(crate) fn require_stopped(&self, pdev: &platform::Device) -> Result { - if self.asc.access(pdev.as_ref())?.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { + pub(crate) fn require_stopped(&self, pdev: &platform::Device>) -> Result { + if self.asc.access(pdev.as_ref())?.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { return Err(EBUSY); } Ok(()) } - pub(crate) fn start_asc(&self, pdev: &platform::Device) -> Result { + pub(crate) fn start_asc(&self, pdev: &platform::Device>) -> Result { self.require_stopped(pdev)?; - let asc = self.asc.access(pdev.as_ref())?; + let asc = self.asc.access(pdev.as_ref())?.as_view(); asc.write32(asc.read32(ASC_CPU_CONTROL) | ASC_CPU_RUN, ASC_CPU_CONTROL); Ok(()) } pub(crate) fn stop_asc(&self) -> Result { - let asc = self.asc.try_access().ok_or(ENODEV)?; + let asc_guard = self.asc.try_access().ok_or(ENODEV)?; + let asc = asc_guard.as_view(); asc.write32(asc.read32(ASC_CPU_CONTROL) & !ASC_CPU_RUN, ASC_CPU_CONTROL); let start = Instant::::now(); while asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN != 0 { @@ -301,7 +262,7 @@ impl Drop for Device { } // ASC and firmware queues must be stopped before the inner GPU vote is relinquished. match self.asc.try_access() { - Some(asc) if asc.read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} + Some(asc) if asc.as_view().read32(ASC_CPU_CONTROL) & ASC_CPU_RUN == 0 => {} _ => { dev_err!(self.dev.as_ref(), "M3 G15G: cannot prove ASC stopped; retaining PMP vote\n"); return; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 25e65c6599084b..2d93823e67d015 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! M3 through the common Asahi GEM, GPUVM and render-node frontend. use core::ops::Range; -use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}}; -use crate::{alloc, driver, drm_gpu::{DrmGpu, DrmGpuParams}, gem, gpu, hw, mmu, queue}; +use core::any::Any; +use kernel::{device::Core, drm, new_mutex, platform, prelude::*, sync::{Arc, Mutex}, uapi}; +use crate::{alloc, driver, gem, gpu, m3::hw, mmu, queue}; -pub(crate) type Shared = Arc>>; -struct ProgressView(Arc); +pub(crate) type Shared = Arc>>; +struct ProgressView(Arc); impl kernel::debugfs::Writer for ProgressView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, completed, last_ns, healthy) = self.0.progress_snapshot(); @@ -14,7 +15,7 @@ impl kernel::debugfs::Writer for ProgressView { } } // Separate endpoint preserves the strict completion-v1 ABI for old guards. -struct ActivityView(Arc); +struct ActivityView(Arc); impl kernel::debugfs::Writer for ActivityView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, epoch, healthy) = self.0.activity_snapshot(); @@ -23,7 +24,7 @@ impl kernel::debugfs::Writer for ActivityView { } } -struct TimingView(Arc); +struct TimingView(Arc); impl kernel::debugfs::Writer for TimingView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (generation, _, _, healthy) = self.0.progress_snapshot(); @@ -33,7 +34,7 @@ impl kernel::debugfs::Writer for TimingView { // Root-only writable debugfs policy; no mutable module-parameter references. // Keep ownership independent of the runtime lock and remove the file on stop. -struct CompletionView(Arc); +struct CompletionView(Arc); impl kernel::debugfs::Writer for CompletionView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { let (us, irqs, sleeps, timers, preparations) = self.0.snapshot(); @@ -53,7 +54,7 @@ impl kernel::debugfs::Reader for CompletionView { } } -struct SubmitOverlapView(Arc); +struct SubmitOverlapView(Arc); impl kernel::debugfs::Writer for SubmitOverlapView { fn write(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { writeln!(f, "{}", u8::from(self.0.cpu_overlap())) @@ -74,137 +75,179 @@ impl kernel::debugfs::Reader for SubmitOverlapView { } } -pub(crate) struct Registered { - registration: Pin>>>>, +/// The started T8122 runtime, as the shared DRM layer sees it ([`gpu::Gpu`]). It is created at +/// probe with the runtime it wraps and handed to the DRM registration, which owns it: dropping it +/// (unbind, or a failed registration) stops the runtime. The debugfs files share its lifetime. +pub(crate) struct Backend { shared: Shared, - health: Arc, + health: Arc, + ids: gpu::SequenceIDs, + scheduler: Arc>, + core_mask: u64, + max_frequency_khz: u32, + soc: &'static crate::m3::m3_soc::Soc, _progress: Pin>>, _activity: Pin>>, - _memory: Pin>>, + _memory: Pin>>, _timing: Pin>>, _completion: Pin>>, _overlap: Pin>>, } -impl Registered { - pub(crate) fn start(pdev: &platform::Device, soc: &'static crate::m3_soc::Soc) -> Result { - let resources = crate::m3_resources::from_device(pdev, soc).inspect_err(|_| { - crate::t8122_start::not_admitted(pdev.as_ref(), soc); - crate::t6031_start::not_admitted(pdev.as_ref(), soc); + +impl Backend { + /// Admits and starts the M3 runtime on `soc` (the T8122), with the DRM device `drm` that the + /// caller registers afterwards. + pub(crate) fn start( + pdev: &platform::Device>, + drm: &driver::AsahiDrmDevice, + soc: &'static crate::m3::m3_soc::Soc, + ) -> Result> { + // The runtime only exists to back its render node: without one (`asahi.m3_expose=0`) + // nothing is started, rather than leaving a running firmware behind a failed probe. + if !crate::m3::m3_board::expose_render_node(soc) { + dev_info!(pdev.as_ref(), "M3: render node not registered on this board (asahi.m3_expose=1 registers it); GPU not started\n"); + return Err(ENODEV); + } + let resources = crate::m3::m3_resources::from_device(pdev, soc).inspect_err(|_| { + crate::m3::t8122_start::not_admitted(pdev.as_ref(), soc); })?; dev_info!(pdev.as_ref(), "M3: resource admission complete\n"); - // An armed T8122 or T6031 start supplies the values its table leaves out. Any other SoC, - // or an unarmed T8122 or T6031, needs a complete table. - let experiment = crate::t8122_start::arm(pdev.as_ref(), soc)?; - let t6031 = crate::t6031_start::arm(pdev.as_ref(), soc)?; - if experiment.is_none() && t6031.is_none() { + // An armed T8122 start supplies the values its table leaves out. An unarmed T8122 + // needs a complete table, which it does not have: it is refused here. + let experiment = crate::m3::t8122_start::arm(pdev.as_ref(), soc)?; + if experiment.is_none() { soc.require_complete(pdev.as_ref())?; } let experiment = experiment.as_ref(); - let t6031 = t6031.as_ref(); let refused = |stage: &'static str| { - move |e: &Error| { - crate::t8122_start::refused(pdev.as_ref(), experiment, stage, *e); - crate::t6031_start::refused(pdev.as_ref(), t6031, stage, *e); - } + move |e: &Error| crate::m3::t8122_start::refused(pdev.as_ref(), experiment, stage, *e) }; - let firmware = crate::m3_firmware::identify_loaded(pdev, soc, resources, experiment, t6031) + let firmware = crate::m3::m3_firmware::identify_loaded(pdev, soc, resources, experiment) .inspect_err(refused("the firmware identity"))?; dev_info!(pdev.as_ref(), "M3: firmware identity accepted\n"); // InitData source and contents, before any GPU register access. - let contents = crate::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment, t6031) + let contents = crate::m3::m3_adt_config::Contents::select(pdev, &firmware, soc, experiment) .inspect_err(refused("the InitData contents"))?; - let device = crate::m3_device::Device::new(pdev, firmware, soc, experiment, t6031) + let device = crate::m3::m3_device::Device::new(pdev, firmware, soc, experiment) .inspect_err(refused("the GPU registers and identity"))?; - device.check_drm(pdev).inspect_err(refused("the DRM setup"))?; let core_mask = device.core_mask(); let max_frequency_khz = 1000 * contents.pstates.reported_max_mhz(); - let mut runtime = crate::m3_runtime::Runtime::new(pdev, device, contents)?; + let mut runtime = crate::m3::m3_runtime::Runtime::new(pdev, drm, device, contents)?; runtime.boot(pdev)?; dev_info!(pdev.as_ref(), "M3: runtime switches: asahi.m3_unlocked_wait={} asahi.m3_timeout_nohang={} resume-after-fault={}\n", - u8::from(crate::m3_params::unlocked_wait()), u8::from(crate::m3_params::timeout_nohang()), - u8::from(crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault))); - let drm = runtime.drm(); + u8::from(crate::m3::m3_params::unlocked_wait()), u8::from(crate::m3::m3_params::timeout_nohang()), + u8::from(crate::m3::m3_params::g15_debug(crate::m3::m3_params::G15Debug::M3ResumeAfterFault))); let health = runtime.health(); - // Health holds no device references. The read-only file owns an Arc - // and is removed with registration; teardown latches failure first. + let state = runtime.rtkit_state(); + // Health holds no device references. The read-only files own an Arc and are removed + // with the backend; teardown latches failure first. let directory = kernel::debugfs::Dir::new(c"asahi-m3"); let progress = KBox::pin_init(directory.read_only_file(c"progress", ProgressView(health.clone())), GFP_KERNEL)?; let activity = KBox::pin_init(directory.read_only_file(c"activity", ActivityView(health.clone())), GFP_KERNEL)?; let memory = KBox::pin_init(directory.read_only_file(c"memory", - crate::agx_memory_stats::View), GFP_KERNEL)?; + crate::m3::agx_memory_stats::View), GFP_KERNEL)?; let timing = KBox::pin_init(directory.read_only_file(c"timing", TimingView(health.clone())), GFP_KERNEL)?; let completion = KBox::pin_init(directory.read_write_file(c"completion_wait", CompletionView(runtime.completion_wait())), GFP_KERNEL)?; let overlap = KBox::pin_init(directory.read_write_file(c"submit_overlap", SubmitOverlapView(runtime.completion_wait())), GFP_KERNEL)?; - let shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; - if !drm.completion.populate(crate::m3_completion::Completion::new(shared.clone())?) { + let scheduler = Arc::new(drm::sched::Scheduler::new(drm.as_ref().as_ref(), 4, 8, 0, 3000, + kernel::c_str!("asahi_m3_sched"))?, GFP_KERNEL)?; + let shared: Shared = Arc::pin_init(new_mutex!(Some(runtime)), GFP_KERNEL)?; + let completion_work = crate::m3::m3_completion::Completion::new(shared.clone())?; + // From here on every failure drops the backend, which stops the runtime. + let backend = Arc::new(Self { shared, health, ids: gpu::SequenceIDs::default(), + scheduler, core_mask, max_frequency_khz, soc, + _progress: progress, _activity: activity, _memory: memory, _timing: timing, + _completion: completion, _overlap: overlap }, GFP_KERNEL)?; + // The completion worker services this runtime from now on; the RTKit callbacks and + // the submission path queue it through the runtime's state. + if !state.completion.populate(completion_work) { return Err(EBUSY); } - let owner = Self { registration: KBox::pin_init(new_mutex!(None), GFP_KERNEL)?, shared: shared.clone(), health: health.clone(), _progress: progress, _activity: activity, _memory: memory, _timing: timing, _completion: completion, _overlap: overlap }; - let scheduler=Arc::new(drm::sched::Scheduler::new(drm.as_ref(),4,8,0,3000,kernel::c_str!("asahi_m3_sched"))?,GFP_KERNEL)?; - let backend: Arc = Arc::new(Backend { shared, health, scheduler, ids: gpu::SequenceIDs::default(), - core_mask, max_frequency_khz, soc }, GFP_KERNEL)?; - if !drm.gpu.populate(crate::drm_gpu::Backend::M3(backend)) { return Err(EBUSY); } - if crate::m3_board::expose_render_node(soc) { - *owner.registration.lock() = Some(drm::driver::Registration::new(&drm, 0)?); - dev_info!(pdev.as_ref(), "M3: firmware ready; common DRM GEM/VM frontend registered\n"); - } else { - dev_info!(pdev.as_ref(), "M3: firmware ready; render node not registered on this board (asahi.m3_expose=1 registers it)\n"); - } - Ok(owner) + Ok(backend) } + + /// Stops the runtime: called on unbind, while the device's resources are still live, and + /// again (doing nothing) when the backend is dropped. pub(crate) fn stop(&self) { self.health.mark_failed(); - drop(self.registration.lock().take()); drop(self.shared.lock().take()); } } -impl Drop for Registered { fn drop(&mut self) { self.stop(); } } -struct Backend { shared: Shared, health: Arc, ids: gpu::SequenceIDs, - scheduler:Arc>, core_mask: u64, max_frequency_khz: u32, - soc: &'static crate::m3_soc::Soc } -impl DrmGpu for Backend { + +impl Drop for Backend { + fn drop(&mut self) { + self.stop(); + } +} + +impl gpu::Gpu for Backend { + fn stats_snapshot(&self) -> Option> { None } + fn as_any(&self) -> &dyn Any { self } + fn arc_as_any(self: Arc) -> Arc { self as Arc } + fn manager(&self) -> Option<&dyn gpu::GpuManager> { None } fn init(&self) -> Result { if self.is_crashed() { Err(ENODEV) } else { Ok(()) } } fn ids(&self) -> &gpu::SequenceIDs { &self.ids } fn is_crashed(&self) -> bool { !self.health.healthy() } - fn update_globals(&self) {} - - fn supports_vm_status(&self) -> bool { true } - fn supports_scheduled_queues(&self)->bool {true} - fn submission_error(&self) -> i32 { if self.is_crashed() { EIO.to_errno() } else { 0 } } - fn params(&self) -> Result { + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { let soc = self.soc; - let masks = crate::m3_board::core_masks(soc, self.core_mask); - Ok(DrmGpuParams { gpu_generation: hw::GpuGen::G15 as u32, - gpu_variant: soc.gpu_variant as u32, gpu_revision: soc.gpu_revision as u32, - chip_id: soc.chip_id, num_dies: 1, num_clusters_total: soc.clusters, - num_cores_per_cluster: soc.cores_per_cluster, - core_masks: masks, max_frequency_khz: self.max_frequency_khz, usc_generation: 3, - gpu_hal_generation: hw::GpuHalGeneration::Legacy as u32, - max_commands_per_submission: crate::file::MAX_COMMANDS_PER_SUBMISSION }) - } - fn user_range(&self) -> Result> { Ok(mmu::UAT_PGSZ as u64..0x2ff_ffff_8000) } - fn unknown_page(&self) -> Result { Ok(0x2ff_ffff_8000) } + let masks = crate::m3::m3_board::core_masks(soc, self.core_mask); + params.gpu_generation = hw::GpuGen::G15 as u32; + params.gpu_variant = soc.gpu_variant as u32; + params.gpu_revision = soc.gpu_revision as u32; + params.chip_id = soc.chip_id; + params.num_dies = 1; + params.num_clusters_total = soc.clusters; + params.num_cores_per_cluster = soc.cores_per_cluster; + for (out, mask) in params.core_masks.iter_mut().zip(masks.iter()) { + *out = (*mask).into(); + } + params.max_frequency_khz = self.max_frequency_khz; + params.usc_generation = 3; + params.gpu_hal_generation = hw::GpuHalGeneration::Legacy as u32; + Ok(()) + } + fn supports_vm_status(&self) -> bool { true } + /// The features the 7.1.12 file layer reported for the M3 runtime: VM status (not its + /// mirror), scheduled queues, and the SoC's optional render and compute features. + fn feature_flags(&self) -> u64 { + let mut flags = uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SCHEDULED_QUEUES as u64; + if self.soc.features.fragment_dependency && crate::m3::m3_params::early_tiling() { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY as u64; + } + if self.soc.features.compute_wide_visibility { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY as u64; + } + flags + } fn base_clock_hz(&self) -> u32 { let value: u64; // SAFETY: read-only architectural counter frequency. unsafe { core::arch::asm!("mrs {x}, CNTFRQ_EL0", x=out(reg) value) }; value as u32 } + fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } + fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..0x2ff_ffff_8000 } + fn tracked_submission(&self) -> bool { true } + /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate). + fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result { + crate::m3::m3_client::admit_vm(dev, params_size, flags) + } fn new_vm(&self, range: Range) -> Result { Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.new_vm(self.ids.vm.next(), range) } fn new_queue(&self, vm: mmu::Vm, _ualloc: Arc>, _ualloc_priv: Arc>, priority: u32, usc_exec_base: u64) -> Result> { - Ok(KBox::new(crate::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, + Ok(KBox::new(crate::m3::m3_submit::Queue::new(self.shared.clone(),self.scheduler.clone(),vm,priority,usc_exec_base, self.soc.features.fragment_dependency)?,GFP_KERNEL)?) } fn map_timestamp_buffer(&self, bo: gem::ObjectRef, range: Range) -> Result { - self.shared.lock().as_ref().ok_or(ENODEV)?.map_timestamp(bo,range) + Option::as_mut(&mut *self.shared.lock()).ok_or(ENODEV)?.map_timestamp(bo,range) } } diff --git a/drivers/gpu/drm/asahi/m3/m3_firmware.rs b/drivers/gpu/drm/asahi/m3/m3_firmware.rs index 9ae509ad959c46..0d2835174c6775 100644 --- a/drivers/gpu/drm/asahi/m3/m3_firmware.rs +++ b/drivers/gpu/drm/asahi/m3/m3_firmware.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -use crate::m3_resources as agx_resources; +use crate::m3::m3_resources as agx_resources; /// A GPU firmware image the runtime can identify. #[derive(Debug)] @@ -30,7 +30,7 @@ pub(crate) static KNOWN_IMAGES: [KnownImage; 2] = [ KnownImage { name: "J514S RTKit-2419.140.12", uuid: None, - stkg_sha256: Some(crate::m3_firmware::TEXT_SHA256), + stkg_sha256: Some(crate::m3::m3_firmware::TEXT_SHA256), initdata_magic: Some(G15_V14_8_3_INITDATA), }, KnownImage { @@ -140,20 +140,6 @@ pub(crate) static T8122_LAYOUT: Layout = Layout { version: "RTKit-2419.140.12.release", }; -/// The T6031 firmware-compat 14.8.3 segment sizes and VAs, from the J516C ADT (iop-gfx-nub0 -/// segment-ranges: TEXT 0x64000 and DATA 0x140000 at the VAs both other layouts use). Neither -/// existing layout matches them. The boot-entropy tag offset is not known (0 stands for it); -/// an armed start searches the loaded text for the tag. The loaded image's version string is -/// not known ("unknown" stands for it). -pub(crate) const T6031_LAYOUT: Layout = Layout { - text_size: 0x64000, - data_size: 0x14_0000, - vas: [0xffff_fc00_0000_0000, 0xffff_fc00_0006_4000], - entropy_tag_offset: 0, - entropy_tag: *b"GKTS\x08\x00\x00\x00", - version: "unknown", -}; - pub(crate) const TEXT_SHA256: [u8; 32] = [ 0x11, 0xe4, 0x9f, 0x75, 0xb, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, 0x80, 0x2, 0x93, 0xd5]; @@ -194,11 +180,10 @@ impl Firmware { #[cfg(not(test))] pub(crate) fn identify_loaded( - pdev: &kernel::platform::Device, - soc: &'static crate::m3_soc::Soc, + pdev: &kernel::platform::Device>, + soc: &'static crate::m3::m3_soc::Soc, resources: agx_resources::Resources, - experiment: Option<&crate::t8122_start::Experiment>, - t6031: Option<&crate::t6031_start::Experiment>, + experiment: Option<&crate::m3::t8122_start::Experiment>, ) -> kernel::error::Result { use kernel::{ bindings, c_str, @@ -220,21 +205,6 @@ pub(crate) fn identify_loaded( let bytes = unsafe { core::slice::from_raw_parts(mapping.ptr(), mapping.size()) }; let mut canonical = KVec::new(); canonical.extend_from_slice(bytes, GFP_KERNEL)?; - // An armed T6031 searches the loaded text for its boot-entropy tag. The layout's offset is - // a placeholder, so the T8122 image table must not see this image. - if let Some(experiment) = t6031 { - experiment.normalize_text(&mut canonical)?; - let mut digest = [0u8; 32]; - // SAFETY: the initialized private copy and distinct digest live for the synchronous - // SHA-256 call. Normalization never writes the loaded firmware. - unsafe { bindings::sha256(canonical.as_ptr(), canonical.len(), digest.as_mut_ptr()) }; - experiment.admit_image(pdev.as_ref(), bytes, &digest)?; - return Ok(Firmware { - resources, - initdata_magic: experiment.initdata_version(), - layout, - }); - } if !normalize_boot_entropy(layout, &mut canonical) { return Err(ENODEV); } @@ -247,7 +217,7 @@ pub(crate) fn identify_loaded( Some(_) => &KNOWN_IMAGES_T8122_EXPERIMENT[..], None => soc.images, }; - let image = crate::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; + let image = crate::m3::m3_board::identify(pdev.as_ref(), bytes, &digest, images).ok_or(ENODEV)?; let initdata_magic = match experiment { Some(experiment) => experiment.initdata_version(), None => image.initdata_magic.ok_or(ENODEV)?, @@ -393,21 +363,6 @@ mod tests { } } - #[test] - fn t6031_layout_is_the_adt_segments_and_matches_neither_other_layout() { - let layout = &T6031_LAYOUT; - assert_eq!((layout.text_size, layout.data_size), (0x64000, 0x140000)); - assert_eq!(layout.vas, [0xffff_fc00_0000_0000, 0xffff_fc00_0000_0000 + 0x64000]); - assert_eq!(layout.entropy_tag, T8122_LAYOUT.entropy_tag); - let mut regions = [Region { base: 0, size: 0 }; 6]; - regions[4].size = layout.text_size; - regions[5].size = layout.data_size; - let resources = Resources { regions, firmware_vas: layout.vas }; - assert!(Firmware::matching_layout(layout, &resources)); - assert!(!Firmware::matching_layout(&T6030_LAYOUT, &resources)); - assert!(!Firmware::matching_layout(&T8122_LAYOUT, &resources)); - } - #[test] fn entropy_normalization_changes_only_the_eight_boot_bytes() { for layout in [&T6030_LAYOUT, &T8122_LAYOUT] { diff --git a/drivers/gpu/drm/asahi/m3/m3_memory.rs b/drivers/gpu/drm/asahi/m3/m3_memory.rs index cbfb771fd08734..5e8018a802e70e 100644 --- a/drivers/gpu/drm/asahi/m3/m3_memory.rs +++ b/drivers/gpu/drm/asahi/m3/m3_memory.rs @@ -8,6 +8,20 @@ enum Backing { Paged(gem::ObjectRef), } +/// Invalidates context 0's non-global translations when dropped. A job's command aliases live in +/// the context-0 lower tree, which has no bind slot on the adopted UAT (7.1.12 gave it the fixed +/// slot 0), so unmapping them invalidates nothing by itself. Declared first in a constructor and +/// last in the job, it drops after every alias. +pub(crate) struct LowerTlbi; + +impl Drop for LowerTlbi { + fn drop(&mut self) { + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); + } +} + pub(crate) struct Buffer { mapping: mmu::KernelMapping, gpu_mapping: Option, @@ -17,18 +31,31 @@ pub(crate) struct Buffer { object: Backing, offset: usize, size: usize, - _coherent_allocation: Option, + _coherent_allocation: Option, +} + +impl Drop for Buffer { + fn drop(&mut self) { + // A GPU view may be a command alias in the context-0 lower tree, which has no bind slot + // to invalidate on unmap: drop it and invalidate context 0 before the backing below is + // freed (7.1.12 invalidated inside the unmap). + if self.gpu_mapping.take().is_some() { + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); + } + } } impl Buffer { - pub(crate) fn at(dev: &driver::AsahiDevice, fw: &mmu::Vm, + pub(crate) fn at(dev: &driver::AsahiDrmDevice, fw: &mmu::Vm, gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, gpu_shared: bool) -> Result { Self::at_prot(dev,fw,gpu,address,size, if gpu_shared {prot::PROT_GPU_FW_SHARED_RW} else {prot::PROT_FW_SHARED_RW}, prot::PROT_GPU_SHARED_RW) } - pub(crate) fn at_prot(dev: &driver::AsahiDevice, fw: &mmu::Vm, + pub(crate) fn at_prot(dev: &driver::AsahiDrmDevice, fw: &mmu::Vm, gpu: Option<(&mmu::Vm,u64)>, address: Option, size: usize, protection: prot::Prot, gpu_protection: prot::Prot) -> Result { let offset=(address.or(gpu.map(|(_,a)|a)).unwrap_or(0)&0x3fff) as usize; @@ -39,18 +66,18 @@ impl Buffer { // normal case here and is not reported; only a failed fallback is. // SAFETY: M3 runtime owns the bound platform device for the entire // diagnostic; all mappings retire before normal DMA allocation Drop. - let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_bound()},allocated,GFP_KERNEL, + let object=kernel::dma::Coherent::zeroed_slice_with_attrs(unsafe {dev.as_ref().as_ref().as_bound()},allocated,GFP_KERNEL, kernel::dma::attrs::DMA_ATTR_NO_WARN).inspect_err(|_| dev_err!(dev.as_ref(),"M3: cannot allocate {} KiB of GPU memory\n",allocated/1024))?; let pa:usize=object.dma_handle().try_into()?; - let mapping=if let Some(a)=address {fw.map_io(a&!0x3fff,pa,allocated,protection)?} + let mapping=if let Some(a)=address {fw.map_io_m3(a&!0x3fff,pa,allocated,protection)?} else {fw.map_io_in_range(0xffff_fc2d_0000_0000..0xffff_fc2e_0000_0000,pa,allocated,protection)?}; - let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; + let gpu_mapping=if let Some((vm,a))=gpu {Some(vm.map_io_m3(a&!0x3fff,pa,allocated,gpu_protection)?)} else {None}; Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Coherent(object),offset,size, - _coherent_allocation:Some(crate::agx_memory_stats::Allocation::coherent(allocated)?)}) + _coherent_allocation:Some(crate::m3::agx_memory_stats::Allocation::coherent(allocated)?)}) } /// Allocate owned tiler storage in the driver-reserved GPU range. Keep the /// old allocation mapped until the replacement has been fully allocated. - pub(crate) fn tiler(dev:&driver::AsahiDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { + pub(crate) fn tiler(dev:&driver::AsahiDrmDevice,fw:&mmu::Vm,vm:&mmu::Vm,size:usize)->Result { if size==0 || size.checked_add(mmu::UAT_PGMSK).is_none() {return Err(EINVAL);} // Large render targets can require 128 MiB of tiler storage. Engines // and firmware address it through UAT, so physically contiguous DMA @@ -62,7 +89,7 @@ impl Buffer { mmu::UAT_PGSZ as u64,prot::PROT_GPU_FW_SHARED_RW,true)?; // Growth retains the cached GPU-only policy of the original TPC and // tilemap allocations; the firmware alias remains shared/uncached. - let gpu_protection=prot::Prot::from_pte(crate::m3_pass_layout::Access::GpuCachedRw.pte()); + let gpu_protection=prot::Prot::from_pte(crate::m3::m3_pass_layout::Access::GpuCachedRw.pte()); let gpu_mapping=Some(object.map_into_range(vm,0x74_0000_0000..0x78_0000_0000, mmu::UAT_PGSZ as u64,gpu_protection,true)?); Ok(Self {mapping,gpu_mapping,gpu_protection,object:Backing::Paged(object),offset:0,size,_coherent_allocation:None}) @@ -73,7 +100,7 @@ impl Buffer { pub(crate) fn map_gpu_view(&mut self,vm:&mmu::Vm)->Result { let address=self.gpu_va()?&!0x3fff; match &mut self.object { - Backing::Coherent(object)=>vm.map_io(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), + Backing::Coherent(object)=>vm.map_io_m3(address,object.dma_handle().try_into()?,object.size(),self.gpu_protection), Backing::Paged(object)=>object.map_at(vm,address,self.gpu_protection,true), } } @@ -82,7 +109,10 @@ impl Buffer { fn cpu_ptr(&mut self)->Result<*mut u8> { match &mut self.object { Backing::Coherent(object)=>Ok(object.as_mut_ptr().cast::()), - Backing::Paged(object)=>Ok(object.cached_vmap()?.as_mut_ptr()), + Backing::Paged(object)=>{ + use kernel::io::IoBase; + Ok(object.vmap()?.as_view().as_ptr().cast::()) + } } } @@ -124,7 +154,7 @@ impl Buffer { if offset.checked_add(4).ok_or(EOVERFLOW)?>self.size {return Err(ERANGE);} let ptr=self.cpu_ptr()?; // SAFETY: checked span in retained CPU storage; no aligned access is required. - unsafe {crate::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; + unsafe {crate::m3::m3_consistent_perf::store_packed(ptr.add(self.offset+offset),value)}; Ok(()) } pub(crate) fn read(&mut self,offset:usize,data:&mut[u8])->Result { diff --git a/drivers/gpu/drm/asahi/m3/m3_params.rs b/drivers/gpu/drm/asahi/m3/m3_params.rs index d34e1f23840eb8..8167c6c3ec05c4 100644 --- a/drivers/gpu/drm/asahi/m3/m3_params.rs +++ b/drivers/gpu/drm/asahi/m3/m3_params.rs @@ -139,7 +139,7 @@ unsafe extern "C" fn get_atomic_param( } fn parse_render_batch_override(text: &str) -> Option { - parse_u64(text).filter(|value| *value <= crate::m3_pass_layout::SLOTS as u64) + parse_u64(text).filter(|value| *value <= crate::m3::m3_pass_layout::SLOTS as u64) } /// Runtime tuning is opt-in: zero keeps the existing boot parameter and its @@ -202,20 +202,58 @@ pub(crate) fn render_internal_timestamps()->bool { M3_RENDER_INTERNAL_TIMESTAMPS.load(Ordering::Relaxed)!=0 } +// The runtime's boot-time module parameters. On 7.1.12 they were `module!` parameters; they +// keep their names, and exist only with the T8122 runtime built in. + +/// `asahi.m3_render_batch_size`: bounded ordered render batch size (1..16). +static M3_RENDER_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_render_batch_size", M3_RENDER_BATCH_SIZE, parse_u64); +/// `asahi.m3_compute_batch_size`: bounded ordered compute batch size (1..16). +static M3_COMPUTE_BATCH_SIZE: AtomicU64 = AtomicU64::new(1); +m3_param!("m3_compute_batch_size", M3_COMPUTE_BATCH_SIZE, parse_u64); +/// `asahi.m3_early_tiling`: opt-in fragment-only stage dependencies. +static M3_EARLY_TILING: AtomicU64 = AtomicU64::new(0); +m3_param!("m3_early_tiling", M3_EARLY_TILING, parse_u64); + +/// `asahi.m3_early_tiling` is set. +pub(crate) fn early_tiling() -> bool { + M3_EARLY_TILING.load(Ordering::Relaxed) != 0 +} + +fn parse_expose(text: &str) -> Option { + match text.trim() { + "-1" => Some(EXPOSE_AUTO), + other => parse_u64(other).filter(|v| *v <= 1), + } +} +const EXPOSE_AUTO: u64 = u64::MAX; +/// `asahi.m3_expose`: M3 render node registration: -1 = auto (on the boards where the runtime +/// is validated), 0 = never, 1 = always. +static M3_EXPOSE: AtomicU64 = AtomicU64::new(EXPOSE_AUTO); +m3_param!("m3_expose", M3_EXPOSE, parse_expose); + +/// `asahi.m3_expose`: `None` for auto, else whether to register the render node. +pub(crate) fn expose() -> Option { + match M3_EXPOSE.load(Ordering::Relaxed) { + EXPOSE_AUTO => None, + v => Some(v != 0), + } +} + /// Snapshot once per batch. A live change affects later publications, never the /// storage, ordering or retirement checks of work already on the firmware queues. pub(crate) fn render_batch_size() -> usize { let override_size = M3_RENDER_BATCH_OVERRIDE.load(Ordering::Relaxed); let size = if override_size == 0 { - *crate::module_parameters::m3_render_batch_size.value() as usize + M3_RENDER_BATCH_SIZE.load(Ordering::Relaxed) as usize } else { override_size as usize }; - size.clamp(1, crate::m3_pass_layout::SLOTS) + size.clamp(1, crate::m3::m3_pass_layout::SLOTS) } fn parse_compute_batch_override(text: &str) -> Option { - parse_u64(text).filter(|value| *value <= crate::m3_compute_storage::SLOTS as u64) + parse_u64(text).filter(|value| *value <= crate::m3::m3_compute_storage::SLOTS as u64) } /// Independently tune the existing adjacent-compute path. Zero preserves the @@ -229,11 +267,11 @@ m3_param!("m3_compute_batch_override", M3_COMPUTE_BATCH_OVERRIDE, pub(crate) fn compute_batch_size() -> usize { let override_size = M3_COMPUTE_BATCH_OVERRIDE.load(Ordering::Relaxed); let size = if override_size == 0 { - *crate::module_parameters::m3_compute_batch_size.value() as usize + M3_COMPUTE_BATCH_SIZE.load(Ordering::Relaxed) as usize } else { override_size as usize }; - size.clamp(1, crate::m3_compute_storage::SLOTS) + size.clamp(1, crate::m3::m3_compute_storage::SLOTS) } /// `asahi.g15_debug`: G15 bring-up bits, see [`G15Debug`]. @@ -320,15 +358,6 @@ fn parse_backend(text: &str) -> Option { static M3_BACKEND: AtomicU64 = AtomicU64::new(BACKEND_AUTO); m3_param!("m3_backend", M3_BACKEND, parse_backend); -/// The T6030 GPU backend to start. -pub(crate) enum T6030Backend { - /// The serialized M3 runtime (m3_drm / m3_runtime). - Runtime, - /// The G15 GpuManager backend (g15_probe). - Manager, - /// No backend: the probe stops without touching the GPU. - Off, -} /// `asahi.m3_initdata` values. const INITDATA_AUTO: u64 = 0; @@ -418,7 +447,7 @@ pub(crate) fn unlocked_wait() -> bool { } /// `asahi.m3_ut_engagement=0|1`: Globals +0x7d0, the gate of the firmware's frequency-feedback -/// cap. Unset: the SoC's default ([`crate::m3_soc::Soc::ut_engagement`]). +/// cap. Unset: the SoC's default ([`crate::m3::m3_soc::Soc::ut_engagement`]). static M3_UT_ENGAGEMENT: AtomicU64 = AtomicU64::new(UT_ENGAGEMENT_UNSET); const UT_ENGAGEMENT_UNSET: u64 = u64::MAX; fn parse_ut_engagement(text: &str) -> Option { @@ -427,7 +456,7 @@ fn parse_ut_engagement(text: &str) -> Option { m3_param!("m3_ut_engagement", M3_UT_ENGAGEMENT, parse_ut_engagement); /// The frequency-feedback gate for `soc`, and whether `asahi.m3_ut_engagement` set it. -pub(crate) fn ut_engagement(soc: &crate::m3_soc::Soc) -> (u32, bool) { +pub(crate) fn ut_engagement(soc: &crate::m3::m3_soc::Soc) -> (u32, bool) { match M3_UT_ENGAGEMENT.load(Ordering::Relaxed) { UT_ENGAGEMENT_UNSET => (soc.ut_engagement, false), v => (v as u32, true), @@ -443,14 +472,14 @@ static M3_PIPELINE_DEPTH: AtomicU64 = AtomicU64::new(0); m3_param!("m3_pipeline_depth", M3_PIPELINE_DEPTH, parse_pipeline_depth, 0o644, Some(get_atomic_param)); fn parse_pipeline_depth(text: &str) -> Option { - parse_u64(text).filter(|v| *v<=crate::m3_pass_layout::SLOTS as u64) + parse_u64(text).filter(|v| *v<=crate::m3::m3_pass_layout::SLOTS as u64) } /// Effective depth; changing it never releases already published pass slots. -pub(crate) fn pipeline_depth(soc: &crate::m3_soc::Soc) -> usize { +pub(crate) fn pipeline_depth(soc: &crate::m3::m3_soc::Soc) -> usize { match M3_PIPELINE_DEPTH.load(Ordering::Relaxed) { - 0 => crate::m3_runtime::policy::default_depth(crate::t8122_start::is_t8122(soc)), - v => v.clamp(1,crate::m3_pass_layout::SLOTS as u64) as usize, + 0 => crate::m3::m3_runtime::policy::default_depth(crate::m3::t8122_start::is_t8122(soc)), + v => v.clamp(1,crate::m3::m3_pass_layout::SLOTS as u64) as usize, } } @@ -464,9 +493,9 @@ static M3_RENDER_BATCH_BUDGET_US: AtomicU64 = AtomicU64::new(u64::MAX); m3_param!("m3_render_batch_budget_us", M3_RENDER_BATCH_BUDGET_US, parse_u64, 0o644, Some(get_atomic_param)); /// The render batch GPU-time budget in ns, or `None` without one. -pub(crate) fn render_batch_budget_ns(soc: &crate::m3_soc::Soc) -> Option { +pub(crate) fn render_batch_budget_ns(soc: &crate::m3::m3_soc::Soc) -> Option { let value=M3_RENDER_BATCH_BUDGET_US.load(Ordering::Relaxed); - let us=if value==u64::MAX {crate::m3_runtime::policy::default_budget_us(crate::t8122_start::is_t8122(soc))} else {value}; + let us=if value==u64::MAX {crate::m3::m3_runtime::policy::default_budget_us(crate::m3::t8122_start::is_t8122(soc))} else {value}; match us { 0 => None, us => Some(us.saturating_mul(1000)), @@ -486,12 +515,12 @@ fn parse_retire_mmio(text: &str) -> Option { /// `asahi.m3_retire_mmio`, writable at runtime: the GPU registers the host reads after every /// job before retiring it. Bit 0 reads the engine-busy registers (SGX+0xc020/0xc120), bit 1 /// selects and reads each cluster's fault bank (0xd800/0xd8c0), bit 2 reads the performance -/// state (0xe01000). Unset: the SoC's default ([`crate::m3_soc::Soc::retire_mmio`]). +/// state (0xe01000). Unset: the SoC's default ([`crate::m3::m3_soc::Soc::retire_mmio`]). static M3_RETIRE_MMIO: AtomicU64 = AtomicU64::new(RETIRE_MMIO_UNSET); m3_param!("m3_retire_mmio", M3_RETIRE_MMIO, parse_retire_mmio, 0o644, Some(get_atomic_param)); /// The `asahi.m3_retire_mmio` mask for `soc`. -pub(crate) fn retire_mmio(soc: &crate::m3_soc::Soc) -> u64 { +pub(crate) fn retire_mmio(soc: &crate::m3::m3_soc::Soc) -> u64 { match M3_RETIRE_MMIO.load(Ordering::Relaxed) { RETIRE_MMIO_UNSET => soc.retire_mmio, mask => mask, @@ -563,7 +592,7 @@ pub(crate) fn thermal_hot_param() -> (u64, bool) { /// The runtime starts only on the boards the T8122 table allows (the M3 MacBook Airs), with /// `auto` and `runtime` alike: it has not run on any T8122, so nothing forces it onto another /// board. The G15 manager backend is T6030-only. -pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { +pub(crate) fn t8122_backend(pdev: &platform::Device>) -> bool { let dev = pdev.as_ref(); match M3_BACKEND.load(Ordering::Relaxed) { BACKEND_OFF => { @@ -577,7 +606,7 @@ pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { ); false } - _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T8122) => true, + _ if crate::m3::m3_board::runtime_validated_board(&crate::m3::m3_soc::T8122) => true, _ => { dev_info!( dev, @@ -588,57 +617,27 @@ pub(crate) fn t8122_backend(pdev: &platform::Device) -> bool { } } -/// Select the T6030 GPU backend from `asahi.m3_backend`, and say which one and why. -/// -/// `auto` starts the runtime backend on the boards it is validated on (the M3 Pro MacBook -/// Pros). On other T6030 boards it starts nothing until the runtime is validated there; -/// `asahi.m3_backend=runtime` starts it anyway. -pub(crate) fn t6030_backend(pdev: &platform::Device) -> T6030Backend { - let dev = pdev.as_ref(); - match M3_BACKEND.load(Ordering::Relaxed) { - BACKEND_RUNTIME => { - dev_info!(dev, "M3: asahi.m3_backend=runtime\n"); - T6030Backend::Runtime - } - BACKEND_MANAGER => { - dev_info!(dev, "M3: asahi.m3_backend=manager: starting the G15 manager backend\n"); - T6030Backend::Manager - } - BACKEND_OFF => { - dev_info!(dev, "M3: asahi.m3_backend=off: no GPU backend started\n"); - T6030Backend::Off - } - _ if crate::m3_board::runtime_validated_board(&crate::m3_soc::T6030) => T6030Backend::Runtime, - _ => { - dev_info!( - dev, - "M3: asahi.m3_backend=auto: no GPU backend on this board: the runtime backend is not validated here yet (asahi.m3_backend=runtime starts it, asahi.m3_backend=manager starts the G15 manager backend)\n" - ); - T6030Backend::Off - } - } -} fn parse_t8122_number(text: &str) -> Option { - Some(crate::t8122_knobs::parse_number(text)) + Some(crate::m3::t8122_knobs::parse_number(text)) } fn parse_t8122_fender(text: &str) -> Option { - Some(crate::t8122_knobs::parse_fender(text)) + Some(crate::m3::t8122_knobs::parse_fender(text)) } fn parse_t8122_clkgen(text: &str) -> Option { - Some(crate::t8122_knobs::parse_clkgen(text)) + Some(crate::m3::t8122_knobs::parse_clkgen(text)) } fn parse_t8122_sgx_setup(text: &str) -> Option { - Some(crate::t8122_knobs::parse_sgx_setup(text)) + Some(crate::m3::t8122_knobs::parse_sgx_setup(text)) } // The T8122 start experiment (`t8122_start`). Each parameter is stored as given, unset or with a // value its parser does not accept (`t8122_knobs`), so that the experiment can refuse rather // than fall back to a default. -use crate::t8122_knobs::UNSET as T8122_UNSET; +use crate::m3::t8122_knobs::UNSET as T8122_UNSET; /// `asahi.t8122_start=1`: start the M3 runtime on a T8122 with the start experiment's values. /// Only with a boot loader that handed the GPU over. @@ -667,21 +666,21 @@ static T8122_PSTATE_CAP: AtomicU64 = AtomicU64::new(T8122_UNSET); m3_param!("t8122_pstate_cap", T8122_PSTATE_CAP, parse_t8122_number); /// `asahi.t8122_hwdata_418c=0|1`: on a T8122 with the G15 14.8.3 firmware, 1 (the default) -/// writes the 20 words at HwDataA +0x418c ([`crate::t8122_hwdata`]); 0 leaves the +/// writes the 20 words at HwDataA +0x418c ([`crate::m3::t8122_hwdata`]); 0 leaves the /// range zero, for an A/B test. Readable in sysfs. -static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::t8122_hwdata::DEFAULT); -m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::t8122_hwdata::parse, 0o444, +static T8122_HWDATA_418C: AtomicU64 = AtomicU64::new(crate::m3::t8122_hwdata::DEFAULT); +m3_param!("t8122_hwdata_418c", T8122_HWDATA_418C, crate::m3::t8122_hwdata::parse, 0o444, Some(get_atomic_param)); /// Whether `asahi.t8122_hwdata_418c` asks for the HwDataA +0x418c words (the default). pub(crate) fn t8122_hwdata_418c() -> bool { - crate::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) + crate::m3::t8122_hwdata::enabled(T8122_HWDATA_418C.load(Ordering::Relaxed)) } /// The T8122 start-experiment parameters of this boot, as given. -pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { +pub(crate) fn t8122_params() -> crate::m3::t8122_knobs::Raw { let get = |p: &AtomicU64| p.load(Ordering::Relaxed); - crate::t8122_knobs::Raw { + crate::m3::t8122_knobs::Raw { start: get(&T8122_START), initdata_version: get(&T8122_INITDATA_VERSION), fender: get(&T8122_FENDER), @@ -692,209 +691,3 @@ pub(crate) fn t8122_params() -> crate::t8122_knobs::Raw { pstate_cap: get(&T8122_PSTATE_CAP), } } - -fn parse_t6031_number(text: &str) -> Option { - Some(crate::t6031_knobs::parse_number(text)) -} - -fn parse_t6031_fender(text: &str) -> Option { - Some(crate::t6031_knobs::parse_fender(text)) -} - -fn parse_t6031_clkgen(text: &str) -> Option { - Some(crate::t6031_knobs::parse_clkgen(text)) -} - -fn parse_t6031_sgx_setup(text: &str) -> Option { - Some(crate::t6031_knobs::parse_sgx_setup(text)) -} - -fn parse_t6031_mcache(text: &str) -> Option { - Some(crate::t6031_knobs::parse_mcache(text)) -} - -fn parse_t6031_aic_swint(text: &str) -> Option { - Some(crate::t6031_knobs::parse_aic_swint(text)) -} - -fn parse_t6031_slot7(text: &str) -> Option { - Some(crate::t6031_knobs::parse_slot7(text)) -} - -fn parse_t6031_gifaf(text: &str) -> Option { - Some(crate::t6031_knobs::parse_gifaf(text)) -} - -fn parse_t6031_slot(text: &str) -> Option { - Some(crate::t6031_knobs::parse_slot(text)) -} - -fn parse_t6031_ane(text: &str) -> Option { - Some(crate::t6031_knobs::parse_ane(text)) -} - -fn parse_t6031_mtr(text: &str) -> Option { - Some(crate::t6031_knobs::parse_mtr(text)) -} - -fn parse_t6031_fw_words(text: &str) -> Option { - Some(crate::t6031_knobs::parse_fw_words(text)) -} - -fn parse_t6031_hwdata_object(text: &str) -> Option { - Some(crate::t6031_knobs::parse_hwdata_object(text)) -} - -fn parse_t6031_rev_id(text: &str) -> Option { - Some(crate::t6031_knobs::parse_rev_id(text)) -} - -fn parse_t6031_tristate(text: &str) -> Option { - Some(crate::t6031_knobs::parse_tristate(text)) -} - -fn parse_t6031_tag(text: &str) -> Option { - Some(crate::t6031_knobs::parse_tag(text)) -} - -// The T6031 start (`t6031_start`). Each parameter is stored as given, unset or with a value its -// parser does not accept (`t6031_knobs`), so the start can refuse rather than fall back. -use crate::t6031_knobs::UNSET as T6031_UNSET; - -/// `asahi.t6031_start=1`: one attempt to start the M3 runtime on a T6031. Without it the probe -/// is the same refusal as before. -static T6031_START: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_start", T6031_START, parse_t6031_number); -/// `asahi.t6031_image_hash`: the first 8 bytes of the loaded text SHA-256, big-endian. -static T6031_IMAGE_HASH: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_image_hash", T6031_IMAGE_HASH, parse_t6031_number); -/// `asahi.t6031_tag=search` or a text offset of the boot-entropy tag. -static T6031_TAG: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_tag", T6031_TAG, parse_t6031_tag); -/// `asahi.t6031_initdata_version=`: the InitData version given to the T6031 firmware. -static T6031_INITDATA_VERSION: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_initdata_version", T6031_INITDATA_VERSION, parse_t6031_number); -/// `asahi.t6031_pstate_cap=N`: the highest performance state the T6031 firmware may use. -static T6031_PSTATE_CAP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_pstate_cap", T6031_PSTATE_CAP, parse_t6031_number); -/// `asahi.t6031_fender=rule|adt`: the Fender window size. -static T6031_FENDER: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_fender", T6031_FENDER, parse_t6031_fender); -/// `asahi.t6031_clkgen=e5c|e1c|none`: the GPU clock-generator IO mapping. -static T6031_CLKGEN: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_clkgen", T6031_CLKGEN, parse_t6031_clkgen); -/// `asahi.t6031_sgx_setup=t6030|none`: the SGX write made before the firmware starts. -static T6031_SGX_SETUP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_sgx_setup", T6031_SGX_SETUP, parse_t6031_sgx_setup); -/// `asahi.t6031_mcache=none|x2|x4|x8`. -static T6031_MCACHE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mcache", T6031_MCACHE, parse_t6031_mcache); -/// `asahi.t6031_aic_swint=page|reg`. -static T6031_AIC_SWINT: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_aic_swint", T6031_AIC_SWINT, parse_t6031_aic_swint); -/// `asahi.t6031_slot7=none|rule`. -static T6031_SLOT7: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot7", T6031_SLOT7, parse_t6031_slot7); -/// `asahi.t6031_gifaf=none|fender`. -static T6031_GIFAF: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_gifaf", T6031_GIFAF, parse_t6031_gifaf); -/// `asahi.t6031_slot21=none` or a physical address. -static T6031_SLOT21: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot21", T6031_SLOT21, parse_t6031_slot); -/// `asahi.t6031_slot24=none` or a physical address. -static T6031_SLOT24: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot24", T6031_SLOT24, parse_t6031_slot); -/// `asahi.t6031_slot28=none` or a physical address. -static T6031_SLOT28: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_slot28", T6031_SLOT28, parse_t6031_slot); -/// `asahi.t6031_ane=none|t6030|t8122` or a physical address. -static T6031_ANE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_ane", T6031_ANE, parse_t6031_ane); -/// `asahi.t6031_io_drop`: a mask of default slots to leave out. -static T6031_IO_DROP: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_io_drop", T6031_IO_DROP, parse_t6031_number); -/// `asahi.t6031_mtr=t6030|t8122|adt|none`. -static T6031_MTR: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr", T6031_MTR, parse_t6031_mtr); -/// `asahi.t6031_mtr_fast_die`: overrides the fast-die MTR mask. -static T6031_MTR_FAST_DIE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr_fast_die", T6031_MTR_FAST_DIE, parse_t6031_number); -/// `asahi.t6031_mtr_alarm`: overrides the alarm MTR mask. -static T6031_MTR_ALARM: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_mtr_alarm", T6031_MTR_ALARM, parse_t6031_number); -/// `asahi.t6031_unit_mask_a=`: HwDataB unit mask A (+0x17c0). -static T6031_UNIT_MASK_A: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_unit_mask_a", T6031_UNIT_MASK_A, parse_t6031_number); -/// `asahi.t6031_unit_mask_b=`: HwDataB unit mask B (+0x17c8). -static T6031_UNIT_MASK_B: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_unit_mask_b", T6031_UNIT_MASK_B, parse_t6031_number); -/// `asahi.t6031_hwb_454`: HwDataB +0xa2c. -static T6031_HWB_454: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_454", T6031_HWB_454, parse_t6031_number); -/// `asahi.t6031_hwb_b20`: HwDataB +0xb20, the core slots. -static T6031_HWB_B20: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_b20", T6031_HWB_B20, parse_t6031_number); -/// `asahi.t6031_hwb_17b8`: HwDataB +0x17b8. -static T6031_HWB_17B8: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_17b8", T6031_HWB_17B8, parse_t6031_number); -/// `asahi.t6031_hwb_1818`: HwDataB +0x1818. -static T6031_HWB_1818: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwb_1818", T6031_HWB_1818, parse_t6031_number); -/// `asahi.t6031_fw_words=none|t8122`. -static T6031_FW_WORDS: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_fw_words", T6031_FW_WORDS, parse_t6031_fw_words); -/// `asahi.t6031_hwdata_object=aligned|fixed`. -static T6031_HWDATA_OBJECT: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_hwdata_object", T6031_HWDATA_OBJECT, parse_t6031_hwdata_object); -/// `asahi.t6031_gpu_core`: must be the table's G15C core type, 24. -static T6031_GPU_CORE: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_gpu_core", T6031_GPU_CORE, parse_t6031_number); -/// `asahi.t6031_rev_id=auto` or 1 to 6. -static T6031_REV_ID: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_rev_id", T6031_REV_ID, parse_t6031_rev_id); -/// `asahi.t6031_csafr=auto|on|off`. -static T6031_CSAFR: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_csafr", T6031_CSAFR, parse_t6031_tristate); -/// `asahi.t6031_power_cap_mw`: the power target while the boot loader's model is a stand-in. -static T6031_POWER_CAP_MW: AtomicU64 = AtomicU64::new(T6031_UNSET); -m3_param!("t6031_power_cap_mw", T6031_POWER_CAP_MW, parse_t6031_number); - -/// The T6031 start parameters of this boot, as given. The image UUID has no module parameter. -pub(crate) fn t6031_params() -> crate::t6031_knobs::Raw { - let get = |p: &AtomicU64| p.load(Ordering::Relaxed); - crate::t6031_knobs::Raw { - start: get(&T6031_START), - image_uuid: crate::t6031_knobs::UuidParam::Unset, - image_hash: get(&T6031_IMAGE_HASH), - tag: get(&T6031_TAG), - initdata_version: get(&T6031_INITDATA_VERSION), - pstate_cap: get(&T6031_PSTATE_CAP), - fender: get(&T6031_FENDER), - clkgen: get(&T6031_CLKGEN), - sgx_setup: get(&T6031_SGX_SETUP), - mcache: get(&T6031_MCACHE), - aic_swint: get(&T6031_AIC_SWINT), - slot7: get(&T6031_SLOT7), - gifaf: get(&T6031_GIFAF), - slot21: get(&T6031_SLOT21), - slot24: get(&T6031_SLOT24), - slot28: get(&T6031_SLOT28), - ane: get(&T6031_ANE), - io_drop: get(&T6031_IO_DROP), - mtr: get(&T6031_MTR), - mtr_fast_die: get(&T6031_MTR_FAST_DIE), - mtr_alarm: get(&T6031_MTR_ALARM), - unit_mask_a: get(&T6031_UNIT_MASK_A), - unit_mask_b: get(&T6031_UNIT_MASK_B), - hwb_454: get(&T6031_HWB_454), - hwb_b20: get(&T6031_HWB_B20), - hwb_17b8: get(&T6031_HWB_17B8), - hwb_1818: get(&T6031_HWB_1818), - fw_words: get(&T6031_FW_WORDS), - hwdata_object: get(&T6031_HWDATA_OBJECT), - gpu_core: get(&T6031_GPU_CORE), - rev_id: get(&T6031_REV_ID), - csafr: get(&T6031_CSAFR), - power_cap_mw: get(&T6031_POWER_CAP_MW), - } -} diff --git a/drivers/gpu/drm/asahi/m3/m3_pass.rs b/drivers/gpu/drm/asahi/m3/m3_pass.rs index 273d2701c3f143..94c2172e33279b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pass.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pass.rs @@ -2,12 +2,12 @@ //! Private storage for one retired/unpublished G15 render pass. A Pass retains //! all its buffers and mappings until Render retires the shared queues. use kernel::prelude::*; -use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_pass_layout as layout}; +use crate::{driver,mmu,pgtable::prot,m3::m3_memory::Buffer,m3::m3_pass_layout as layout}; use layout::{Field,Space}; pub(crate) struct Pass {buffers:KVec} impl Pass { - pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, slot:usize,clusters:u32,aliases:&mut KVec)->Result { let mut buffers=KVec::with_capacity(layout::COUNT,GFP_KERNEL)?; for field in layout::FIELDS { @@ -28,7 +28,7 @@ impl Pass { pub(crate) fn get_mut(&mut self,field:Field)->&mut Buffer {&mut self.buffers[field as usize]} /// Caller must prove retirement, and must not clear a prepared sibling. /// Dependency commands are rewritten completely by the sync encoder. - pub(crate) fn reset(&mut self,geometry:&crate::agx_render::Geometry)->Result { + pub(crate) fn reset(&mut self,geometry:&crate::m3::agx_render::Geometry)->Result { // Tiler arenas grow to retain the largest prior target. The firmware // descriptors use the current geometry's strides and layer count, so // only that prefix is addressable by this pass. Clear it on every use, diff --git a/drivers/gpu/drm/asahi/m3/m3_pool.rs b/drivers/gpu/drm/asahi/m3/m3_pool.rs index 979c1a37bb202b..ab50ab7eca8a17 100644 --- a/drivers/gpu/drm/asahi/m3/m3_pool.rs +++ b/drivers/gpu/drm/asahi/m3/m3_pool.rs @@ -3,7 +3,7 @@ //! GPU mappings and counters outlive this initialization and remain owned by //! Render/Compute. Callers must never invoke this on a live or replayed pool. use kernel::prelude::*; -use crate::{m3_memory::Buffer, m3_pool_layout as wire, m3_queue_layout::FirmwareVa}; +use crate::{m3::m3_memory::Buffer, m3::m3_pool_layout as wire, m3::m3_queue_layout::FirmwareVa}; pub(crate) struct Owners { pub(crate) backing: core::ops::Range, pub(crate) pages: usize, diff --git a/drivers/gpu/drm/asahi/m3/m3_render.rs b/drivers/gpu/drm/asahi/m3/m3_render.rs index c2c67f5c3aa5ac..e7a02982e96f12 100644 --- a/drivers/gpu/drm/asahi/m3/m3_render.rs +++ b/drivers/gpu/drm/asahi/m3/m3_render.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT use kernel::prelude::*; -use crate::m3_queue_layout as queue; -use crate::m3_sync_layout as sync; -use crate::m3_state_layout as state; -use crate::m3_scene_layout as scene; -use crate::{m3_pass::{Pass,ViewOwner},m3_pass_layout::{self as pass_layout,Field as P,Space}}; -use crate::{m3_fragment_command as fragment, m3_tiler_command as tiler, m3_render_sequence as sequence, m3_init_layout::Region}; -use crate::{m3_parameter_layout as parameter, m3_pool_layout::GpuRegion}; -use crate::{driver,mmu,pgtable::prot,m3_memory::Buffer,m3_shared_layout as s,agx_uapi}; -use crate::m3_timeline as timeline; +use crate::m3::m3_queue_layout as queue; +use crate::m3::m3_sync_layout as sync; +use crate::m3::m3_state_layout as state; +use crate::m3::m3_scene_layout as scene; +use crate::{m3::m3_pass::{Pass,ViewOwner},m3::m3_pass_layout::{self as pass_layout,Field as P,Space}}; +use crate::{m3::m3_fragment_command as fragment, m3::m3_tiler_command as tiler, m3::m3_render_sequence as sequence, m3::m3_init_layout::Region}; +use crate::{m3::m3_parameter_layout as parameter, m3::m3_pool_layout::GpuRegion}; +use crate::{driver,mmu,pgtable::prot,m3::m3_memory::Buffer,m3::m3_shared_layout as s,m3::agx_uapi}; +use crate::m3::m3_timeline as timeline; use s::{PB_FIRST,PB_GROUPS,PB_BLOCKS_PER_GROUP,PB_BLOCK_SIZE}; const PB_BLOCKS:u32=(PB_GROUPS*PB_BLOCKS_PER_GROUP) as u32; pub(crate) const PB_PAGES:u32=PB_BLOCKS*4; @@ -38,7 +38,9 @@ pub(crate) struct Render { /// GPU clusters of the board: tiler accelerators (TPC storage) and preemption buffers. clusters:u32, /// The register lists of the SoC's GPU (`m3_soc::Soc::registers`). - registers:crate::m3_compute_layout::RegisterSet, + registers:crate::m3::m3_compute_layout::RegisterSet, + /// Last: drops after the pass aliases. + _lower_tlbi:crate::m3::m3_memory::LowerTlbi, } // Queue allocations belong to Render, not to a pass. Keep their fixed arenas // and firmware/shared mapping permissions from the qualified allocation map. @@ -51,9 +53,9 @@ use s::{GPU_CONTEXT,JOB_LIST,BUFFER_MANAGER,PARAMETER_PAGE_LIST, PARAMETER_BLOCK_LIST,NOTIFIER,BLOCK_CONTROL,MANAGER_MISC}; const MANAGER_SCENE_COUNT: u32 = 48; const TA_POOL_COUNTER: usize = 11; -const UMA_POOLS: [crate::m3_pool::Owners; 2] = [ - crate::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, - crate::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, +const UMA_POOLS: [crate::m3::m3_pool::Owners; 2] = [ + crate::m3::m3_pool::Owners { backing: 0..8, pages: 8, table: 9, manager: 10, counter: 11, table_index: 0 }, + crate::m3::m3_pool::Owners { backing: 12..20, pages: 20, table: 21, manager: 22, counter: 23, table_index: 1 }, ]; const FRAGMENT_POOL_COUNTER: usize = 23; // Retained producer identifier for cross-pass dependencies, G15 / V14_8_3. @@ -65,8 +67,9 @@ const RENDER_QUEUES: [QueueObjects; 2] = [ scratch: s::FRAGMENT_SCRATCH, tiler: false, uuid: 0x3d0000 }, ]; impl Render { - pub(crate) fn new(dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, - registers:crate::m3_compute_layout::RegisterSet)->Result { + pub(crate) fn new(dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm,stats:Region,clusters:u32, + registers:crate::m3::m3_compute_layout::RegisterSet)->Result { + let lower_tlbi=crate::m3::m3_memory::LowerTlbi; let binding=uat.bind(vm)?; let mut objects=KVec::new();let mut aliases=KVec::new(); for index in 0..s::COUNT { @@ -91,14 +94,14 @@ impl Render { tiler_bytes.resize(tiler::SIZE,0u8,GFP_KERNEL)?; let mut fragment_bytes=KVec::new(); fragment_bytes.resize(fragment::SIZE,0u8,GFP_KERNEL)?; - let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers}; + let mut job=Self{passes,init_bm,stats,sequence_bytes,tiler_bytes,fragment_bytes,objects,_aliases:aliases,_binding:binding,last_progress_ns:0,draw:0,heads:[0;2],cached_views:KVec::new(),slot:0,batch_count:0,early_count:0,stage_timestamps:[true;2],base:0,checkpoint:(0,[0;2],0,0,0,0),clusters,registers,_lower_tlbi:lower_tlbi}; // Lists and the matching manager are built before InitBM publication. job.initialize_parameter_buffer()?; for pool in &UMA_POOLS { pool.initialize(&mut job.objects)?; } job.initialize_state()?; job.initialize_queues()?; dev_info!(dev.as_ref(),"M3 native render storage initialized, PB={} MiB\n",PB_BLOCKS as usize*PB_BLOCK_SIZE/(1024*1024)); - crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + crate::m3::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); unsafe {core::arch::asm!("isb",options(nostack,preserves_flags))}; Ok(job) } @@ -223,7 +226,7 @@ impl Render { // Prepare each private command after its retired slot is reset. The // reusable host staging allocation is copied before the next slot is built. fn prepare_tiler(&mut self,r:agx_uapi::UapiRenderCommand,draw:u64)->Result { - use crate::m3_compute_layout::GpuVa; + use crate::m3::m3_compute_layout::GpuVa; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); @@ -247,7 +250,7 @@ impl Render { user_timestamps:[fw(P::TilerUserStart)?,fw(P::TilerUserEnd)?],preemption:[gpu(P::Preemption0)?,gpu(P::Preemption1)?,gpu(P::Preemption2)?], tilemap:gpu(P::Tilemap)?,tpc:gpu(P::Tpc)?,tpc_bytes:g.tpc_bytes,heap:gpu(P::HeapMetadata)?, scene_user:gpu(P::SceneUser)?,initial_scene_entry,geometry:g,page_count:PB_PAGES, - vdm:crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, + vdm:crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?, multisample:r.multisample_control,ppp:r.ppp_control,process_empty_tiles:r.flags&2!=0, }; value.encode(&mut self.tiler_bytes).map_err(|_|EINVAL)?; @@ -290,8 +293,8 @@ impl Render { Ok(()) } /// Pass geometry; TPC storage is per tiler accelerator, one per GPU cluster. - fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { - use crate::agx_render::{Geometry,Utile}; + fn geometry(r:agx_uapi::UapiRenderCommand,clusters:u32)->Result { + use crate::m3::agx_render::{Geometry,Utile}; let x=if r.utile_width==16 {Utile::Pixels16} else {Utile::Pixels32}; let y=if r.utile_height==16 {Utile::Pixels16} else {Utile::Pixels32}; let mut g=Geometry::new_layered(u32::from(r.width),u32::from(r.height),x,y,clusters,r.layers).map_err(|_|EINVAL)?; @@ -299,7 +302,7 @@ impl Render { Ok(g) } fn prepare_fragment(&mut self,r:agx_uapi::UapiRenderCommand,usc:u64,draw:u64)->Result { - use crate::{m3_compute_layout::GpuVa,agx_render_state::{Program,DepthStencil}}; + use crate::{m3::m3_compute_layout::GpuVa,m3::agx_render_state::{Program,DepthStencil}}; let pass=&self.passes[self.slot]; let shared=|index:usize| queue::FirmwareVa::new(self.objects[index].va()).map_err(|_|EINVAL); let fw=|field:P| queue::FirmwareVa::new(pass.get(field).va()).map_err(|_|EINVAL); @@ -355,7 +358,7 @@ impl Render { self.passes[self.slot].get_mut(sequence).u64(start+sequence::USER_TIMESTAMP_POINTER,pointer)?; self.passes[self.slot].get_mut(sequence).u64(end+sequence::USER_TIMESTAMP_POINTER,pointer)?; } - crate::agx_memory::publish();Ok(()) + crate::m3::agx_memory::publish();Ok(()) } fn at(base:usize,index:usize)->usize {(base+index)%pass_layout::SLOTS} /// GPU time of the batch of `count` passes starting at pass slot `base`. @@ -416,13 +419,13 @@ impl Render { /// Start a batch in pass slots `base..`. With `overlap`, earlier batches may still run: then /// the VM must already be bound, no storage may grow and the slots must not be in flight /// (the caller keeps the in-flight passes plus this batch within SLOTS). - pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDevice,uat:&mmu::Uat,vm:&mmu::Vm, - commands:&[crate::m3_submit::Command],base:usize,overlap:bool)->Result { + pub(crate) fn begin_batch(&mut self,dev:&driver::AsahiDrmDevice,uat:&mmu::Uat,vm:&mmu::Vm, + commands:&[crate::m3::m3_submit::Command],base:usize,overlap:bool)->Result { if base>=pass_layout::SLOTS || (!overlap && !self.complete()?) {return Err(EBUSY);} if overlap { if !self._binding.matches(vm) {return Err(EBUSY);} for (index,&command) in commands.iter().enumerate() { - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; let g=Self::geometry(r,self.clusters)?; let slot=Self::at(base,index); if g.tpc_bytes>self.passes[slot].get(P::Tpc).size() as u64 @@ -433,9 +436,9 @@ impl Render { self.draw.checked_add(commands.len() as u64).ok_or(EOVERFLOW)?; // Validate every pass before changing any queue or completion state. for &command in commands { - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; Self::geometry(r,self.clusters)?; - crate::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; + crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_|EINVAL)?; } if !self._binding.matches(vm) { let next=if let Some(index)=self.cached_views.iter().position(|v|v.binding.matches(vm)) { @@ -477,7 +480,7 @@ impl Render { // already-grown buffers are reusable on the next submission. for (index,&command) in commands.iter().enumerate() { let slot=Self::at(base,index); - let crate::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=command else {return Err(EINVAL);}; let g=Self::geometry(r,self.clusters)?; for (field,base,bytes) in [(P::Tpc,s::TPC_LABEL,g.tpc_bytes),(P::Tilemap,s::TILEMAP_LABEL,g.tilemap_bytes)] { let size=usize::try_from(bytes).map_err(|_|EOVERFLOW)?; @@ -504,7 +507,7 @@ impl Render { for (stage,pointers) in [s::TA_POINTERS,s::FRAGMENT_POINTERS].into_iter().enumerate() { self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); Ok(()) } /// Append only into caller-reserved free pass slots. With overlap, WRITE updates @@ -533,7 +536,7 @@ impl Render { // Retain the conservative stage-scope permission. Only a pass after the // first of its batch may overlap its predecessor's stages. Explicit // start timestamps keep the original full-order execution boundary. - let early=predecessor.is_none() && self.batch_count>0 && *crate::module_parameters::m3_early_tiling.value()!=0 + let early=predecessor.is_none() && self.batch_count>0 && crate::m3::m3_params::early_tiling() && r.flags&(1<<5)!=0 && r.vertex_timestamps.start.handle==0 && r.fragment_timestamps.start.handle==0; if early {self.early_count+=1;} @@ -585,15 +588,15 @@ impl Render { self.objects[ring].u64(usize::from(self.heads[stage])*8,address)?; self.heads[stage]=(self.heads[stage]+1)%0x500; } - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); self.objects[pointers].u32(queue::WRITE,u32::from(self.heads[stage]))?; } - crate::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) + crate::m3::agx_memory::publish();crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync();Ok(()) } /// Emitted only after both events, stamp/queue checks, clear faults, /// idle engines and consumed firmware pipes have been checked by Runtime. /// The existing TA/fragment descriptors request completion stamp flushes. - pub(crate) fn progress(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn progress(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Keep summaries sparse at high throughput, but emit at least once a // second while completed batches advance. Batched draw counts can skip // multiples of 128; a count-only gate can starve the watchdog observer. @@ -654,7 +657,7 @@ impl Render { || self.objects[pointers].read_u32(queue::READ)?!=u32::from(head) {return Ok(false);} } Ok(true) } - pub(crate) fn log(&mut self,dev:&driver::AsahiDevice)->Result { + pub(crate) fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result { // Retain allocator state as well as payloads: an exhausted parameter // pool can stall TA without producing an MMU fault. for (index,bytes) in [(BLOCK_CONTROL,64),(s::BM_COUNTER,64),(MANAGER_MISC,64),(BUFFER_MANAGER,172)] { diff --git a/drivers/gpu/drm/asahi/m3/m3_resources.rs b/drivers/gpu/drm/asahi/m3/m3_resources.rs index bfc94fdfa0ca8d..7d343052500182 100644 --- a/drivers/gpu/drm/asahi/m3/m3_resources.rs +++ b/drivers/gpu/drm/asahi/m3/m3_resources.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! M3 GPU resources admitted against the measured boot contract (J514S on T6030). -pub(crate) use crate::agx_resources::{Region, Resources}; +pub(crate) use crate::m3::agx_resources::{Region, Resources}; use kernel::{bindings, c_str, io::resource::Resource, of, prelude::*}; @@ -11,13 +11,13 @@ pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result = node.get_property(c_str!("compatible"))?; let compatible = compatible.strip_suffix(b"\0").ok_or(EINVAL)?; - if !crate::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { + if !crate::m3::m3_soc::SOCS.iter().any(|soc| compatible == soc.gpu.as_bytes()) { return Err(EINVAL); } let names: KVec = node.get_property(c_str!("memory-region-names"))?; @@ -44,8 +44,8 @@ pub(crate) fn reserved_resource(node: &of::Node, name: &CStr) -> Result, - soc: &crate::m3_soc::Soc, + pdev: &kernel::platform::Device>, + soc: &crate::m3::m3_soc::Soc, ) -> kernel::error::Result { - crate::m3_board::admit(pdev, soc) + crate::m3::m3_board::admit(pdev, soc) } diff --git a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs index 3e8cbbfa937817..be81c4a4e9d928 100644 --- a/drivers/gpu/drm/asahi/m3/m3_rtkit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_rtkit.rs @@ -3,18 +3,17 @@ //! RTKit callbacks for the identified J514S firmware. The firmware advertises //! its crash storage by physical address inside the reserved data segment. -use crate::agx_resources::Region; +use crate::m3::agx_resources::Region; use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use kernel::{ c_str, impl_has_hr_timer, device::Core, sync::Completion, - io::mem::{Mem, MemFlag}, - iosys_map::IoSysMapRef, + io::{mem::{Mem, MemFlag}, IoSysMap, Region as IoRegion, SysMem}, platform, prelude::*, soc::apple::rtkit, - sync::{aref::ARef, Arc, ArcBorrow}, + sync::{aref::ARef, Arc, ArcBorrow, SetOnce}, time::{Delta, Monotonic, hrtimer::{HrTimer, HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode}}, }; @@ -40,7 +39,7 @@ impl EventWait { pin_init!(EventWait { count: AtomicU64::new(0), wait_us: AtomicU64::new(100), - overlap_cpu: AtomicBool::new(crate::m3_params::unlocked_wait()), + overlap_cpu: AtomicBool::new(crate::m3::m3_params::unlocked_wait()), cpu_preparations: AtomicU64::new(0), sleeps: AtomicU64::new(0), timer_wakes: AtomicU64::new(0), @@ -139,6 +138,9 @@ pub(crate) struct State { /// The firmware sends one after a command's completion processing. pub(crate) event_messages: AtomicU64, pub(crate) events: Arc, + /// The completion worker, published once the runtime it services exists. The DRM device + /// carries no driver data for it on 7.2, so the runtime's own state owns it. + pub(crate) completion: SetOnce>, } /// Shared status contains no device references, so DRM files may retain it @@ -147,8 +149,8 @@ pub(crate) struct Health { crashed: AtomicBool, mapped: AtomicBool, failed: AtomicBool, - progress: crate::agx_host_progress::Progress, - pub(crate) timing: crate::agx_timing_stats::Stats, + progress: crate::m3::agx_host_progress::Progress, + pub(crate) timing: crate::m3::agx_timing_stats::Stats, } impl Health { @@ -181,7 +183,7 @@ impl Health { } impl State { - pub(crate) fn new(dev: &platform::Device, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { + pub(crate) fn new(dev: &platform::Device>, drm: crate::driver::AsahiDevRef, data: Region) -> Result> { Ok(Arc::new( Self { dev: dev.into(), @@ -191,12 +193,13 @@ impl State { crashed: AtomicBool::new(false), mapped: AtomicBool::new(false), failed: AtomicBool::new(false), - progress: crate::agx_host_progress::Progress::new(), - timing: crate::agx_timing_stats::Stats::new(), + progress: crate::m3::agx_host_progress::Progress::new(), + timing: crate::m3::agx_timing_stats::Stats::new(), }, GFP_KERNEL)?, claimed: AtomicBool::new(false), event_messages: AtomicU64::new(0), events: EventWait::new()?, + completion: SetOnce::new(), }, GFP_KERNEL, )?) @@ -227,8 +230,18 @@ impl rtkit::Buffer for FirmwareBuffer { fn iova(&self) -> Result { Ok(self.physical) } - fn buf(&mut self) -> Result> { - self.mapping.iosys_map(self.offset, self.size) + fn buf(&mut self) -> Result> { + // The view of 7.1.12's `Mem::iosys_map`: the bounded span inside the retained WB + // mapping, as normal memory. + if self.offset.checked_add(self.size).ok_or(EOVERFLOW)? > self.mapping.size() { + return Err(EINVAL); + } + // SAFETY: The span lies entirely inside the retained mapping (checked above). + let base = unsafe { self.mapping.ptr().add(self.offset) }; + let region = IoRegion::ptr_try_from_raw_parts_mut(base, self.size)?; + // SAFETY: The span is mapped kernel memory for as long as `self.mapping`, which the + // returned view borrows through `&mut self`. + Ok(IoSysMap::Sys(unsafe { SysMem::new(region) })) } } @@ -244,15 +257,12 @@ impl rtkit::Operations for Operations { // Wake the serialized worker even if the firmware can no longer send // its usual completion notification. Never take the runtime lock here. state.events.record(); - crate::m3_completion::queue(&state.drm); + crate::m3::m3_completion::queue(&state); dev_err!( state.dev.as_ref(), "M3 G15S: firmware crashed, retained crashlog bytes={}\n", crashlog.map_or(0, |b| b.len()) ); - if let Some(bytes) = crashlog { - crate::t6031_start::capture_crashlog(state.dev.as_ref(), bytes); - } } fn shmem_map( @@ -275,7 +285,7 @@ impl rtkit::Operations for Operations { return Err(EBUSY); } let node = state.dev.as_ref().of_node().ok_or(ENODEV)?; - let resource = crate::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; + let resource = crate::m3::m3_resources::reserved_resource(&node,c_str!("fw-data"))?; if resource.start() != state.data.base || resource.size() != state.data.size { return Err(EINVAL); } @@ -314,7 +324,7 @@ impl rtkit::Operations for Operations { ::ktime_get(), state.event_messages.load(Ordering::Acquire)); } - crate::m3_completion::queue(&state.drm); + crate::m3::m3_completion::queue(&state); return; } dev_warn!( diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index 08885f591c1406..b12fc73f9f248d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -2,19 +2,19 @@ use core::{mem::ManuallyDrop, sync::atomic::Ordering}; use kernel::{device::Core, platform, prelude::*, soc::apple::rtkit, sync::Arc, time::{Instant, Monotonic, Delta, delay::fsleep}}; -use crate::{driver, mmu, m3_device::Device, m3_config::Config, m3_rtkit}; +use crate::{driver, mmu, m3::m3_device::Device, m3::m3_config::Config, m3::m3_rtkit}; #[path = "m3_batch_policy.rs"] pub(crate) mod policy; -const _: () = assert!(policy::SLOTS == crate::m3_pass_layout::SLOTS); +const _: () = assert!(policy::SLOTS == crate::m3::m3_pass_layout::SLOTS); -enum NativeJob {Compute(crate::m3_compute::Compute),Render(crate::m3_render::Render)} +enum NativeJob {Compute(crate::m3::m3_compute::Compute),Render(crate::m3::m3_render::Render)} /// Completion-worker outcome. pub(crate) enum Service {Idle,Progress,Waiting(u64)} /// The published batch: its commands in order, as (packet, command index). struct Batch { - entries: KVec<(Arc,usize)>, - index: usize, kind: usize, first: crate::m3_submit::Command, + entries: KVec<(Arc,usize)>, + index: usize, kind: usize, first: crate::m3::m3_submit::Command, preparation_ns: i64, previous_events: u64, expected_events: u64, start: Instant, measure: bool, doorbell_counter_ns: i64, /// Render: first pass slot and the batch's last draw number. @@ -40,16 +40,16 @@ struct GeometryTiming { } impl NativeJob { fn complete(&mut self)->Result {match self {Self::Compute(j)=>j.complete(),Self::Render(j)=>j.complete()}} - fn log(&mut self,dev:&driver::AsahiDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} + fn log(&mut self,dev:&driver::AsahiDrmDevice)->Result {match self {Self::Compute(j)=>j.log(dev),Self::Render(j)=>j.log(dev)}} fn batch_gpu_span(&mut self)->Result<[u64;2]> {match self {Self::Compute(j)=>j.batch_gpu_span(),Self::Render(j)=>j.batch_gpu_span()}} } struct Inner { transport: rtkit::RtKit, state: Arc, jobs: KVec, - packets: KVec>, + packets: KVec>, /// Packets not yet (fully) published: (packet, next command index), most urgent first. - pending: KVec<(Arc,usize)>, + pending: KVec<(Arc,usize)>, /// Published batches, oldest first. More than one only with asahi.m3_pipeline_depth > 1. active: KVec, gpu_pending: bool, @@ -57,14 +57,14 @@ struct Inner { active_power_samples: u32, fault_captured: bool, timing: [[i64;7];2], - render_batches: [u64;crate::m3_pass_layout::SLOTS], + render_batches: [u64;crate::m3::m3_pass_layout::SLOTS], geometry: [GeometryTiming; 32], geometry_overflow: u64, /// Whether a T8122 job verdict line was already emitted for the current job, so the /// execute-path fallback does not add a second one. T8122 only. t8122_verdict: bool, /// The step the firmware boot sequence has reached, for the T8122 boot verdict. T8122 only. - boot_step: crate::t8122_start::BootStep, + boot_step: crate::m3::t8122_start::BootStep, /// Packets that joined a batch started by an earlier packet (debug summary). coalesced_packets: u64, coalesced_render_stamps: u64, @@ -76,6 +76,9 @@ struct Inner { render_initialized: bool, config: Config, uat: mmu::Uat, + /// The firmware's view of the host window (the shared L2 the firmware walks), kept in step + /// with the kernel VM's shadow window before the firmware is given new kernel addresses. + window: crate::g15::uat::Window, drm: driver::AsahiDevRef, device: Device, } @@ -86,10 +89,10 @@ impl Inner { self.state.health.mark_failed(); if self.fault_captured { return; } self.fault_captured = true; - #[cfg(CONFIG_DEV_COREDUMP)] - if let Err(error) = self.config.fault_snapshot(&self.drm, primary, self.gpu_pending) { - dev_err!(self.drm.as_ref(), "M3 fault snapshot failed: {:?}\n", error); - } + // 7.1.12 also published a devcoredump of the firmware objects here; 7.2's kernel crate + // has no devcoredump binding, so the first error is only logged. + dev_err!(self.drm.as_ref(), "M3: first GPU failure: {:?} (GPU work pending: {})\n", + primary, self.gpu_pending); } /// Record a batch that failed or never retired. Mark the GPU failed first, @@ -97,15 +100,12 @@ impl Inner { /// behind a device that still reports itself healthy. fn fail(&mut self, index: usize, vm: &mmu::Vm, primary: Error) { self.capture_fault(primary); - if crate::t8122_start::is_t8122(self.device.soc()) { + if crate::m3::t8122_start::is_t8122(self.device.soc()) { let pstate=self.device.pstate_register().ok(); // The job's first GPU start timestamp: nonzero means the firmware dispatched it, // whatever the power state reads at the timeout. let start_stamp=self.jobs[index].batch_gpu_span().ok().map(|span| span[0]); - crate::t8122_start::job_failed_verdict(self.drm.as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::job_failed_verdict(self.drm.as_ref(), primary, self.state.health.crashed()); + crate::m3::t8122_start::job_failed_verdict(self.drm.as_ref().as_ref(),primary,start_stamp,pstate,self.state.health.crashed()); self.t8122_verdict=true; } let events=self.state.event_messages.load(Ordering::Acquire); @@ -116,7 +116,7 @@ impl Inner { dev_err!(self.drm.as_ref(), "M3 engine diagnostics failed: {:?}\n", e); } let _=self.config.log_recovery_state(&self.drm,events); - if crate::m3_params::g15_debug(crate::m3_params::G15Debug::M3ResumeAfterFault) { + if crate::m3::m3_params::g15_debug(crate::m3::m3_params::G15Debug::M3ResumeAfterFault) { if let Err(e)=self.resume_experiment(index, vm) { dev_err!(self.drm.as_ref(),"M3: resume-after-fault: stopped by {:?}\n",e); } @@ -203,10 +203,10 @@ impl Inner { } /// Hardware completion is separate from fence signaling: cancellation can win the /// signal race while the DMA guard must survive until its final published batch retires. - fn packet_active(&self, packet:&Arc)->bool { + fn packet_active(&self, packet:&Arc)->bool { self.active.iter().any(|b|b.entries.iter().any(|(p,_)|Arc::ptr_eq(p,packet))) } - fn retire_packets(&mut self,entries:&[(Arc,usize)]) { + fn retire_packets(&mut self,entries:&[(Arc,usize)]) { for (packet,_) in entries { let pending=self.pending.iter().any(|(p,_)|Arc::ptr_eq(p,packet)); if policy::packet_retired(pending,self.packet_active(packet)) {packet.retired();} @@ -221,37 +221,37 @@ impl Inner { } } /// Signal every packet of `entries` (packets that continue past the batch included). - fn finish_all(entries:&[(Arc,usize)],result:Result) { + fn finish_all(entries:&[(Arc,usize)],result:Result) { for (packet,_) in entries {packet.finish(result);} } /// Choose the next batch from the front of `pending`: the front packet's next commands of /// one engine, and for render also the leading render commands of following packets of the /// same VM, up to the ordered-batch limit. - fn next_batch(&self,max:usize)->Result,usize)>> { + fn next_batch(&self,max:usize)->Result,usize)>> { let mut entries=KVec::new(); let Some((packet,next))=self.pending.first() else {return Ok(entries);}; - let mixed=policy::mixed_prefix(crate::m3_params::mixed_batch(), + let mixed=policy::mixed_prefix(crate::m3::m3_params::mixed_batch(), self.device.soc().chip_id,self.render_initialized,self.active.is_empty(), - packet.commands[*next..].iter().map(|c|matches!(c,crate::m3_submit::Command::Render{..}))); + packet.commands[*next..].iter().map(|c|matches!(c,crate::m3::m3_submit::Command::Render{..}))); if mixed>0 { for index in *next..*next+mixed {entries.push((packet.clone(),index),GFP_KERNEL)?;} return Ok(entries); } - let render=matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}); + let render=matches!(packet.commands[*next],crate::m3::m3_submit::Command::Render{..}); let limit=if render { // The first render draw initializes the shared buffer manager (InitBM) and opens // both queues. Publish it alone: appending later draws first advances the shared // manager and event counters it starts from, and on J613 the second pass's // fragment then fails with firmware error event 4 (draw=2 batch=2). let started=self.jobs.iter().any(|job| matches!(job,NativeJob::Render(j) if j.started())); - if started {crate::m3_params::render_batch_size().min(max).max(1)} else {1} + if started {crate::m3::m3_params::render_batch_size().min(max).max(1)} else {1} } else { - crate::m3_params::compute_batch_size() - .clamp(1,crate::m3_compute_storage::SLOTS) + crate::m3::m3_params::compute_batch_size() + .clamp(1,crate::m3::m3_compute_storage::SLOTS) }; - let same_engine=|c:&crate::m3_submit::Command| render==matches!(c,crate::m3_submit::Command::Render{..}); + let same_engine=|c:&crate::m3::m3_submit::Command| render==matches!(c,crate::m3::m3_submit::Command::Render{..}); let per_pass=self.pass_cost.get(packet.vm.id()); - let budget=crate::m3_params::render_batch_budget_ns(self.device.soc()); + let budget=crate::m3::m3_params::render_batch_budget_ns(self.device.soc()); let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); for (position,(candidate,start)) in self.pending.iter().enumerate() { // Later packets join only render batches, only from their first command, and only @@ -272,19 +272,19 @@ impl Inner { /// Whether the front pending work may be published behind the in-flight batches, and if so /// in which pass slot it starts and how many passes it may use. fn pipeline_slot(&mut self)->Result> { - let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3_pass_layout::SLOTS)));}; + let Some(last)=self.active.last() else {return Ok(Some((0,crate::m3::m3_pass_layout::SLOTS)));}; if !self.render_initialized {return Ok(None);} let Some((packet,next))=self.pending.first() else {return Ok(None);}; - if !matches!(packet.commands[*next],crate::m3_submit::Command::Render{..}) + if !matches!(packet.commands[*next],crate::m3::m3_submit::Command::Render{..}) || self.active.iter().any(|b| b.kind!=0) || !packet.vm.same(&last.entries[0].0.vm) {return Ok(None);} let in_flight:usize=self.active.iter().map(|b|b.entries.len()).sum(); - let Some(room)=policy::pipeline_room(crate::m3_params::pipeline_depth(self.device.soc()), + let Some(room)=policy::pipeline_room(crate::m3::m3_params::pipeline_depth(self.device.soc()), self.active.len(),in_flight,last.base,last.entries.len()) else {return Ok(None);}; // Yield at the next retirement when more urgent work is already waiting. if policy::urgent_waiting(self.pending.iter().map(|(p,_)|p.urgency), last.entries[0].0.urgency) {return Ok(None);} if !policy::within_budget(self.pass_cost.get(packet.vm.id()), - crate::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} + crate::m3::m3_params::render_batch_budget_ns(self.device.soc()),in_flight) {return Ok(None);} if !policy::overlap_ready(self.render_initialized,true,true,true, self.config.pipe_free(0)? && self.config.pipe_free(1)?) {return Ok(None);} Ok(Some(room)) @@ -325,11 +325,8 @@ impl Inner { } Err((error,published))=>{ if published { - if crate::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { - crate::t8122_start::job_setup_failed_verdict(self.drm.as_ref(),error); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) && !self.t8122_verdict { - crate::t6031_start::job_setup_failed_verdict(self.drm.as_ref(), error); + if crate::m3::t8122_start::is_t8122(self.device.soc()) && !self.t8122_verdict { + crate::m3::t8122_start::job_setup_failed_verdict(self.drm.as_ref().as_ref(),error); self.t8122_verdict=true; } // A live ring WRITE or doorbell may have been published: keep packets @@ -355,9 +352,9 @@ impl Inner { progress } /// Prepare and publish one batch. On error, reports whether firmware may see its writes. - fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { - let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Render{..})); - let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3_submit::Command::Compute(_))); + fn publish(&mut self,entries:&[(Arc,usize)],base:usize,overlap:bool)->core::result::Result { + let render=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3::m3_submit::Command::Render{..})); + let compute=entries.iter().any(|(p,i)|matches!(p.commands[*i],crate::m3::m3_submit::Command::Compute(_))); if render && compute {return self.publish_mixed(entries,base,overlap);} let early=|e:Error|(e,false); let (packet,first_index)=(&entries[0].0,entries[0].1); @@ -378,19 +375,19 @@ impl Inner { if !self.packets.iter().any(|q| Arc::ptr_eq(q,p)) {self.packets.push(p.clone(),GFP_KERNEL).map_err(|_|early(ENOMEM))?;} } let found=self.jobs.iter().position(|job| matches!((job,control), - (NativeJob::Compute(_),crate::m3_submit::Command::Compute(_)) | - (NativeJob::Render(_),crate::m3_submit::Command::Render{..}))); + (NativeJob::Compute(_),crate::m3::m3_submit::Command::Compute(_)) | + (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..}))); let index=found.unwrap_or(self.jobs.len()); if found.is_none() { let job=match control { - crate::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), - crate::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Compute(c)=>NativeJob::Compute(crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,c,self.device.soc().registers).map_err(early)?), + crate::m3::m3_submit::Command::Render{..}=>NativeJob::Render(crate::m3::m3_render::Render::new(&self.drm,&self.uat,&packet.vm,self.config.stats_region().map_err(early)?,self.device.soc().clusters,self.device.soc().registers).map_err(early)?), }; self.jobs.push(job,GFP_KERNEL).map_err(|_|early(ENOMEM))?; } else { match (&mut self.jobs[index],control) { - (NativeJob::Render(_),crate::m3_submit::Command::Render{..})=>{}, - (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, + (NativeJob::Render(_),crate::m3::m3_submit::Command::Render{..})=>{}, + (NativeJob::Compute(j),crate::m3::m3_submit::Command::Compute(c))=>j.replay(&self.uat,&packet.vm,c).map_err(early)?, _=>return Err(early(ENOTSUPP)), } } @@ -398,11 +395,11 @@ impl Inner { let prepared=(||->Result { for (slot,(p,next)) in entries.iter().enumerate() { if slot!=0 { - let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; j.append(c)?; } let timestamps=p.timestamp_addresses(*next); - let coalesce=crate::m3_compute_storage::coalesce_stamp_flush( + let coalesce=crate::m3::m3_compute_storage::coalesce_stamp_flush( p.wide_visibility[*next],timestamps,slot+1==batch_count); j.prepare_completion(coalesce)?; j.set_user_timestamps(timestamps)?; @@ -417,8 +414,8 @@ impl Inner { } } let mut coalesced_stamps=0; - let coalesce_render=crate::m3_params::render_stamp_coalesce(); - let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let coalesce_render=crate::m3::m3_params::render_stamp_coalesce(); + let internal_timestamps=crate::m3::m3_params::render_internal_timestamps(); let chip=self.device.soc().chip_id; if let NativeJob::Render(j)=&mut self.jobs[index] { let mut commands=KVec::with_capacity(batch_count,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -426,7 +423,7 @@ impl Inner { j.begin_batch(&self.drm,&self.uat,&packet.vm,&commands,base,overlap).map_err(early)?; let prepared=(||->Result { for (slot,(p,next)) in entries.iter().enumerate() { - let crate::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command,usc}=p.commands[*next] else {return Err(EINVAL);}; let timestamps=p.render_timestamp_addresses(*next); let coalesce=policy::coalesce_render_stamps(coalesce_render,chip, overlap,slot,batch_count,timestamps); @@ -448,6 +445,19 @@ impl Inner { } } let preparation_ns=prepare.elapsed().as_nanos(); + // The batch's kernel objects may have created leaf tables in the host window: link + // them into the firmware's shared L2 before it is told about the batch. A failure + // means the window no longer matches what the firmware may walk: stop scheduling and + // retain the batch, as for a published error. + if let Err(error)=self.window.sync_all(&self.uat) { + return Err((error,true)); + } + // The command aliases live in the context-0 lower tree, which has no bind slot on the + // adopted UAT (7.1.12 gave it the fixed slot 0), so remapping one invalidates nothing: + // drop context 0's non-global entries before the firmware fetches the new ones. + crate::mem::sync(); + crate::mem::tlbi_asid(0); + crate::mem::sync(); self.t8122_verdict=false; let previous_events=self.config.completed_events; let publication_tick=if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) {physical_counter()} else {0}; @@ -470,7 +480,7 @@ impl Inner { }; let measure=crate::debug::debug_enabled(crate::debug::DebugFlags::M3SubmitSummary); let (base,last_draw)=match &self.jobs[index] {NativeJob::Render(j)=>{let (b,_,d)=j.batch();(b,d)},_=>(0,0)}; - Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3_submit::Command::Compute(_))), + Ok(Batch { entries:owned, index, kind:usize::from(matches!(control,crate::m3::m3_submit::Command::Compute(_))), first:control, preparation_ns, previous_events, expected_events, start:Instant::::now(), measure, doorbell_counter_ns:if measure {physical_counter() as i64 * 1000 / 24} else {0}, polling_ns:0, polls:0, stamp_ns:-1, first_event_ns:-1, events_ns:-1, completion_ns:[-1;4],fragment_host_tick:0,fragment_fw_tick:0,fragment_host_pending_tick:0,fragment_fw_pending_tick:0,pipes_idle_tick:0, coalesced_stamps, base, last_draw, publication_tick, mixed:None }) @@ -478,7 +488,7 @@ impl Inner { /// Prepare both idle queues from one packet. Cross-engine changes wait on /// the preceding command's firmware stamp; all commands remain GPU ordered. /// Producer storage and DMA guards survive every partial publication error. - fn publish_mixed(&mut self,entries:&[(Arc,usize)], + fn publish_mixed(&mut self,entries:&[(Arc,usize)], base:usize,overlap:bool)->core::result::Result { let early=|e:Error|(e,false); if overlap || base!=0 || self.device.soc().chip_id!=0x6030 @@ -486,7 +496,7 @@ impl Inner { let packet=&entries[0].0; if entries.iter().any(|(p,_)|!Arc::ptr_eq(packet,p)) {return Err(early(EINVAL));} let prepare=Instant::::now(); - let internal_timestamps=crate::m3_params::render_internal_timestamps(); + let internal_timestamps=crate::m3::m3_params::render_internal_timestamps(); self.active.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; self.jobs.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; self.packets.reserve(1,GFP_KERNEL).map_err(|_|early(ENOMEM))?; @@ -497,8 +507,8 @@ impl Inner { for (p,i) in entries { owned.push((p.clone(),*i),GFP_KERNEL).map_err(|_|early(ENOMEM))?; match p.commands[*i] { - c@crate::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, - crate::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, + c@crate::m3::m3_submit::Command::Render{..}=>render_commands.push(c,GFP_KERNEL).map_err(|_|early(ENOMEM))?, + crate::m3::m3_submit::Command::Compute(c)=>{computes+=1;first_compute.get_or_insert(c);}, } } let renders=render_commands.len(); @@ -514,7 +524,7 @@ impl Inner { let prepared=(||->Result { let control=first_compute.ok_or(EINVAL)?; if existing.is_none() { - let job=crate::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, + let job=crate::m3::m3_compute::Compute::new(&self.drm,&self.uat,&packet.vm, self.config.stats_region()?,control,self.device.soc().registers)?; self.jobs.push(NativeJob::Compute(job),GFP_KERNEL)?; } else { @@ -528,12 +538,12 @@ impl Inner { let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; j.begin_batch(&self.drm,&self.uat,&packet.vm,&render_commands,0,false)?; render_prepared=true; - let mut previous:Option<(bool,crate::m3_sync_layout::CompletionDependency)>=None; + let mut previous:Option<(bool,crate::m3::m3_sync_layout::CompletionDependency)>=None; let mut compute_slot=0; let mut dependencies=0; for (p,i) in entries { match p.commands[*i] { - crate::m3_submit::Command::Render{command,usc}=>{ + crate::m3::m3_submit::Command::Render{command,usc}=>{ let predecessor=previous.filter(|(render,_)|!*render).map(|(_,d)|d); let NativeJob::Render(j)=&mut self.jobs[index] else {return Err(EINVAL);}; let timestamps=p.render_timestamp_addresses(*i); @@ -543,7 +553,7 @@ impl Inner { dependencies+=usize::from(predecessor.is_some()); previous=Some((true,j.completion_dependency()?)); }, - crate::m3_submit::Command::Compute(c)=>{ + crate::m3::m3_submit::Command::Compute(c)=>{ let NativeJob::Compute(j)=&mut self.jobs[compute_index] else {return Err(EINVAL);}; if compute_slot!=0 {j.append(c)?;} if let Some((true,dependency))=previous { @@ -628,14 +638,14 @@ impl Inner { let mut compute_slot=0; for (packet,next) in batch.entries.iter() { match packet.commands[*next] { - crate::m3_submit::Command::Render{command:r,..}=>{ + crate::m3::m3_submit::Command::Render{command:r,..}=>{ let NativeJob::Render(j)=&mut self.jobs[batch.index] else {return Err(EINVAL);}; let s=j.slot_stage_ns(0,mixed.renders,render_slot)?; dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} mixed=1 slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n", vm,render_slot,mixed.renders,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); render_slot+=1; }, - crate::m3_submit::Command::Compute(c)=>{ + crate::m3::m3_submit::Command::Compute(c)=>{ let NativeJob::Compute(j)=&mut self.jobs[mixed.compute_index] else {return Err(EINVAL);}; let ns=j.slot_gpu_ns(compute_slot)?; dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} mixed=1 slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n", @@ -718,10 +728,10 @@ impl Inner { } } } - let power_enabled=crate::m3_params::active_power(); + let power_enabled=crate::m3::m3_params::active_power(); if power_enabled { let since_last=self.active_power_last.map(|last|last.elapsed().as_nanos()); - if crate::m3_active_power::due(power_enabled,self.device.soc().chip_id, + if crate::m3::m3_active_power::due(power_enabled,self.device.soc().chip_id, stamped,self.active_power_samples,since_last) { self.active_power_last=Some(Instant::::now()); self.active_power_samples+=1; @@ -760,23 +770,20 @@ impl Inner { (evented,batch.start.elapsed()>=Delta::from_secs(2),batch.previous_events) }; if stamped && evented { - let retire_mmio=crate::m3_params::retire_mmio(self.device.soc()); + let retire_mmio=crate::m3::m3_params::retire_mmio(self.device.soc()); match self.device.check_idle_parts( - retire_mmio&crate::m3_params::RETIRE_MMIO_BUSY!=0, - retire_mmio&crate::m3_params::RETIRE_MMIO_FAULTS!=0) { + retire_mmio&crate::m3::m3_params::RETIRE_MMIO_BUSY!=0, + retire_mmio&crate::m3::m3_params::RETIRE_MMIO_FAULTS!=0) { Ok(())=>match self.config.pipes_idle() { Ok(true)=>{ if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { let tick=physical_counter(); if let Some(batch)=self.active.first_mut() {batch.pipes_idle_tick=tick;} } - if retire_mmio&crate::m3_params::RETIRE_MMIO_PSTATE!=0 { + if retire_mmio&crate::m3::m3_params::RETIRE_MMIO_PSTATE!=0 { if let Err(e)=self.config.check_pstate(&self.drm,&self.device,"after a job") { - if crate::t8122_start::is_t8122(self.device.soc()) { - crate::t8122_start::cap_violated_verdict(self.drm.as_ref(),e); - self.t8122_verdict=true; - } else if crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::cap_violated_verdict(self.drm.as_ref(), e); + if crate::m3::t8122_start::is_t8122(self.device.soc()) { + crate::m3::t8122_start::cap_violated_verdict(self.drm.as_ref().as_ref(),e); self.t8122_verdict=true; } self.fail_active(e);return Err(e); @@ -862,8 +869,8 @@ impl Inner { if crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming) { let observed_tick=physical_counter(); let (cs,span)=match (&mut self.jobs[index],control) { - (NativeJob::Compute(j),crate::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), - (NativeJob::Render(j),crate::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, + (NativeJob::Compute(j),crate::m3::m3_submit::Command::Compute(c))=>(c.base,j.batch_gpu_span()?), + (NativeJob::Render(j),crate::m3::m3_submit::Command::Render{command:r,..})=>{let (a,b)=j.gpu_ticks(batch.base,batch_count)?;(r.vdm_base,[a,b])}, _=>return Err(EINVAL), }; let valid=batch.publication_tick!=0 && span[0]>=batch.publication_tick && span[1]>=span[0] && observed_tick>=span[1]; @@ -874,14 +881,14 @@ impl Inner { if kind==0 {gpu_ns} else {0},stages[1]); if let NativeJob::Compute(j)=&mut self.jobs[index] { for (slot,(p,next)) in batch.entries.iter().enumerate() { - let crate::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Compute(c)=p.commands[*next] else {return Err(EINVAL);}; let ns=j.slot_gpu_ns(slot)?; dev_info!(self.drm.as_ref(),"M3_COMPUTE_SLOT vm={} slot={} batch={} cdm={:#x} end={:#x} usc={:#x} gpu_ns={}\n",p.vm.id(),slot,batch_count,c.base,c.end,c.usc_base,ns); } } if let NativeJob::Render(j)=&mut self.jobs[index] { for (slot,(p,next)) in batch.entries.iter().enumerate() { - let crate::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; + let crate::m3::m3_submit::Command::Render{command:r,..}=p.commands[*next] else {return Err(EINVAL);}; let s=j.slot_stage_ns(batch.base,batch_count,slot)?; dev_info!(self.drm.as_ref(),"M3_PASS_SLOT vm={} slot={} batch={} vdm={:#x} width={} height={} samples={} ta_ns={} fragment_ns={} gap_ns={}\n",p.vm.id(),slot,batch_count,r.vdm_base,r.width,r.height,r.samples,s[0],s[1],s[2]); } @@ -893,7 +900,7 @@ impl Inner { preparation_ns, active_ns, gpu_ns, stages); let retirement_ns=retire_start.map_or(0,|t|t.elapsed().as_nanos()); if batch.measure { - if let crate::m3_submit::Command::Render {command:r,..}=control { + if let crate::m3::m3_submit::Command::Render {command:r,..}=control { if batch_count>1 {self.geometry_overflow+=1;} else { if let Some(g)=self.geometry.iter_mut().find(|g| g.count==0 || (g.width==r.width && g.height==r.height && g.samples==r.samples)) { @@ -917,9 +924,9 @@ impl Inner { } if crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { match control { - crate::m3_submit::Command::Render {command:r,..}=> + crate::m3::m3_submit::Command::Render {command:r,..}=> dev_info!(self.drm.as_ref(),"M3_PASS kind=render batch={} vdm={:#x} width={} height={} samples={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={}\n",batch_count,r.vdm_base,r.width,r.height,r.samples,preparation_ns,active_ns,gpu_ns,stages[0],stages[1],stages[2]), - crate::m3_submit::Command::Compute(_)=> + crate::m3::m3_submit::Command::Compute(_)=> dev_info!(self.drm.as_ref(),"M3_PASS kind=compute prepare_ns={} active_ns={} gpu_ns={}\n",preparation_ns,active_ns,gpu_ns), } } @@ -927,14 +934,12 @@ impl Inner { t[0]+=1;t[1]+=preparation_ns;t[2]+=active_ns;t[3]+=gpu_ns as i64; for i in 0..3 {t[4+i]+=stages[i] as i64;} // Running totals are in debugfs `timing`; the periodic line is a SubmitTiming diagnostic. - if t[0]==1 && crate::t8122_start::is_t8122(self.device.soc()) { + if t[0]==1 && crate::m3::t8122_start::is_t8122(self.device.soc()) { let span=match &mut self.jobs[index] { NativeJob::Compute(j)=>j.batch_gpu_span().ok(), NativeJob::Render(j)=>j.gpu_ticks(batch.base,batch_count).ok().map(|(a,b)|[a,b]), }; - crate::t8122_start::job_completed_verdict(self.drm.as_ref(),kind,gpu_ns,span); - } else if t[0]==1 && crate::t6031_start::is_t6031(self.device.soc()) { - crate::t6031_start::job_completed_verdict(self.drm.as_ref(), kind, gpu_ns); + crate::m3::t8122_start::job_completed_verdict(self.drm.as_ref().as_ref(),kind,gpu_ns,span); } if t[0]%128==0 && crate::debug::debug_enabled(crate::debug::DebugFlags::SubmitTiming) { dev_info!(self.drm.as_ref(),"M3_TIMING kind={} count={} prepare_ns={} active_ns={} gpu_ns={} ta_ns={} fragment_ns={} gap_ns={} ordinal={}\n",kind,t[0],t[1],t[2],t[3],t[4],t[5],t[6], @@ -971,26 +976,23 @@ impl Inner { pub(crate) struct Runtime { inner: ManuallyDrop> } impl Runtime { #[inline(never)] - pub(crate) fn new(pdev: &platform::Device, device: Device, contents: crate::m3_adt_config::Contents) -> Result { + pub(crate) fn new(pdev: &platform::Device>, drm: &driver::AsahiDrmDevice, device: Device, contents: crate::m3::m3_adt_config::Contents) -> Result { // On an armed T8122, name a setup failure before the GPU coprocessor is started (RTKit // state, mailbox transport, the DRM device). Captured as a Copy bool so the closure does // not borrow `device`, which is moved into Inner below. - let t8122 = crate::t8122_start::is_t8122(device.soc()); - let t6031 = crate::t6031_start::is_t6031(device.soc()); + let t8122 = crate::m3::t8122_start::is_t8122(device.soc()); let coproc_refused = |e: &Error| { - if t8122 { crate::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } - if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), *e); } + if t8122 { crate::m3::t8122_start::coproc_setup_refused(pdev.as_ref(), *e); } }; device.require_stopped(pdev).inspect_err(coproc_refused)?; // Reserve before starting ASC. ENOMEM must not drop firmware owners // while the coprocessor may still access them. Writing the completed // graph into this allocation is infallible. let owner = KBox::::new_uninit(GFP_KERNEL)?; - let drm: driver::AsahiDevRef = kernel::drm::Device::new(pdev.as_ref(), driver::AsahiData::new(pdev, None, true)).inspect_err(coproc_refused)?; + let drm: driver::AsahiDevRef = drm.into(); let state = m3_rtkit::State::new(pdev, drm.clone(), device.firmware().resources.regions[5]).inspect_err(coproc_refused)?; let mut transport = rtkit::RtKit::new(pdev.as_ref(), None, 0, state.clone()).inspect_err(coproc_refused)?; - if crate::m3_adt_config::stop_before_asc(pdev.as_ref()) { - if t6031 { crate::t6031_start::coproc_setup_refused(pdev.as_ref(), ENODEV); } + if crate::m3::m3_adt_config::stop_before_asc(pdev.as_ref()) { return Err(ENODEV); } // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. @@ -1004,18 +1006,22 @@ impl Runtime { Pin::new(&mut transport).start_endpoint(ep)?; } started=true; - // The UAT geometry of the admitted SoC (complete: `Soc::require_complete`). - // SAFETY: the admitted RTKit (J514S on T6030) is awake; no initdata or GPU - // job has been published. Its running PPL handoff owns the peer lock. - let uat=unsafe {mmu::Uat::new_m3_running(&drm,device.soc())}?; + // The firmware's UAT, adopted as the G15 backend adopts it on 7.2: the handoff and + // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as + // every context's upper half, and the host window (root entry 2) through the + // shared L2. The RTKit is awake and no initdata or GPU job has been published. + let (uat,mut window)=adopt_uat(&drm,device.soc(),&device.firmware().resources)?; let config=Config::new(&drm,&uat,device.firmware(),&contents)?; - Ok((uat,config)) + // Link every kernel table the configuration created before the firmware sees an + // address in it, then drop any translation cached before they existed. + window.sync_all(&uat)?; + crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); + Ok((uat,window,config)) })(); - let (uat,config)=match prepared { + let (uat,window,config)=match prepared { Ok(v)=>v, Err(e)=>{ - crate::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); - if t6031 { crate::t6031_start::prepare_verdict(pdev.as_ref(), started, e); } + crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; @@ -1025,34 +1031,43 @@ impl Runtime { return Err(e); } }; - Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, drm, device, - jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3_pass_layout::SLOTS], + Ok(Self { inner: ManuallyDrop::new(owner.write(Inner { transport, state, config, uat, window, drm, device, + jobs:KVec::new(),packets:KVec::new(),pending:KVec::new(),active:KVec::new(),gpu_pending:false,active_power_last:None,active_power_samples:0,fault_captured:false,timing:[[0;7];2],render_batches:[0;crate::m3::m3_pass_layout::SLOTS], geometry:[GeometryTiming::default();32],geometry_overflow:0,coalesced_packets:0,coalesced_render_stamps:0,mixed_batches:0,pass_cost:policy::PassCosts::default(),render_initialized:false,t8122_verdict:false, - boot_step:crate::t8122_start::BootStep::Publish })) }) + boot_step:crate::m3::t8122_start::BootStep::Publish })) }) } pub(crate) fn drm(&self) -> driver::AsahiDevRef { self.inner.drm.clone() } pub(crate) fn health(&self) -> Arc { self.inner.state.health.clone() } + /// The RTKit callbacks' state, which owns the completion worker. + pub(crate) fn rtkit_state(&self) -> Arc { self.inner.state.clone() } pub(crate) fn new_vm(&mut self, id: u64, range: core::ops::Range) -> Result { use crate::util::RangeExt; let reserved_range=0x70_0000_0000..0x80_0000_0000; if range.overlaps(reserved_range.clone()) {return Err(EINVAL);} if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } - let vm=self.inner.uat.new_vm(id, range)?; + // Status and job lifetime tracking: deferred unmaps while accepted work runs + // (7.1.12's M3 VM job lifetime, the shared `mmu::lifetime` on 7.2). + let vm=self.inner.uat.new_vm(id, range)?.with_tracked_status()?; let mut reserved=KVec::new();reserved.push(reserved_range,GFP_KERNEL)?;reserved.push(0x100_8000_0000..0x101_0000_0000,GFP_KERNEL)?;reserved.push(0x10_0000_0000..0x10_0800_0000,GFP_KERNEL)?;reserved.push(0x10_7400_0000..0x10_7402_8000,GFP_KERNEL)?; vm.install_driver_mappings(KVec::new(),reserved)?; Ok(vm) } - pub(crate) fn map_timestamp(&self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { + pub(crate) fn map_timestamp(&mut self,mut bo:crate::gem::ObjectRef,range:core::ops::Range)->Result { if self.inner.gpu_pending { self.inner.state.events.record_cpu_preparation(); } - bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::agx_memory::TIMESTAMP_RANGE, - mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false) + let mapping=bo.map_range_into_range(self.inner.uat.kernel_vm(),range,crate::m3::agx_memory::TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64,mmu::PROT_FW_SHARED_RW,false)?; + // A job of an overlapping batch can reach the firmware without a new publication: + // link a new leaf table now, not only at the next batch. + let inner: &mut Inner=&mut self.inner; + inner.window.sync_all(&inner.uat)?; + Ok(mapping) } /// Queue a packet and return its fence at once. The scheduler can hand over further /// ready jobs up to its credit limit while the GPU runs. Only the completion worker /// publishes: it coalesces render commands of one VM and urgency, and may keep bounded /// same-VM batches in flight after InitBM retirement. VM/engine changes wait for full /// retirement. A burst can therefore share a firmware round trip across packets. - pub(crate) fn submit(shared:&crate::m3_drm::Shared,packet:Arc)->Result { + pub(crate) fn submit(shared:&crate::m3::m3_drm::Shared,packet:Arc)->Result { crate::debug::update_debug_flags(); let mut guard=shared.lock(); let inner=&mut *Option::as_mut(&mut *guard).ok_or(ENODEV)?.inner; @@ -1069,14 +1084,14 @@ impl Runtime { if inner.pending.len()>=policy::MAX_PACKETS {return Err(EBUSY);} inner.pending.reserve(1,GFP_KERNEL)?; inner.pending.insert_within_capacity(at,(packet,0)).map_err(|_|ENOMEM)?; - let drm=inner.drm.clone(); + let state=inner.state.clone(); drop(guard); - crate::m3_completion::queue(&drm); + crate::m3::m3_completion::queue(&state); Ok(()) } /// Withdraw a packet's unpublished remainder on cancellation or timeout. Its active /// commands remain owned until retirement; error signaling cannot release a DMA guard. - pub(crate) fn withdraw(shared:&crate::m3_drm::Shared,packet:&Arc)->bool { + pub(crate) fn withdraw(shared:&crate::m3::m3_drm::Shared,packet:&Arc)->bool { let mut guard=shared.lock(); let Some(runtime)=Option::as_mut(&mut *guard) else {return false;}; let inner: &mut Inner=&mut runtime.inner; @@ -1120,18 +1135,17 @@ impl Runtime { Err(_)=>{inner.fail_pending(EIO);Service::Idle}, } } - pub(crate) fn boot(&mut self, pdev: &platform::Device) -> Result { - let is_t8122 = crate::t8122_start::is_t8122(self.inner.device.soc()); - let is_t6031 = crate::t6031_start::is_t6031(self.inner.device.soc()); - self.inner.boot_step = crate::t8122_start::BootStep::Publish; + pub(crate) fn boot(&mut self, pdev: &platform::Device>) -> Result { + let is_t8122 = crate::m3::t8122_start::is_t8122(self.inner.device.soc()); + self.inner.boot_step = crate::m3::t8122_start::BootStep::Publish; let result = self.boot_inner(pdev); if let Err(error) = result { // On T8122, log the firmware readiness words on every boot failure, so the verdict's // "see M3 firmware readiness" always has a line to point at. The ready-wait timeout in // boot_inner already logs them (on every SoC, as in 12.0), so skip that case here. - let logged = self.inner.boot_step == crate::t8122_start::BootStep::AwaitReady + let logged = self.inner.boot_step == crate::m3::t8122_start::BootStep::AwaitReady && error == ETIMEDOUT; - if (is_t8122 || is_t6031) && !logged { + if is_t8122 && !logged { let inner: &mut Inner = &mut *self.inner; let _ = inner.config.log_ready(&inner.drm); } @@ -1139,18 +1153,13 @@ impl Runtime { } if is_t8122 { let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); - crate::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, - self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); - } - if is_t6031 { - let accepted = result.is_ok() || self.inner.config.ready().unwrap_or(false); - crate::t6031_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, + crate::m3::t8122_start::boot_verdict(pdev.as_ref(), self.inner.device.firmware().initdata_magic, self.inner.boot_step, accepted, self.inner.state.health.crashed(), result); } result } - fn boot_inner(&mut self, pdev: &platform::Device) -> Result { - use crate::t8122_start::BootStep; + fn boot_inner(&mut self, pdev: &platform::Device>) -> Result { + use crate::m3::t8122_start::BootStep; let root = self.inner.config.root(); dev_info!(pdev.as_ref(), "M3: publishing owned initdata {:#x}\n", root); self.inner.boot_step = BootStep::Publish; @@ -1174,7 +1183,7 @@ impl Runtime { inner.config.drain(&inner.drm)?; inner.config.log_ready(&inner.drm)?; if inner.device.soc().chip_id==0x6030 && - (crate::m3_params::active_power() || crate::m3_params::globals_debug_off()) { + (crate::m3::m3_params::active_power() || crate::m3::m3_params::globals_debug_off()) { inner.config.log_globals_debug(&inner.drm)?; } inner.config.check_pstate(&inner.drm, &inner.device, "after boot")?; @@ -1244,3 +1253,22 @@ pub(crate) fn physical_counter() -> u64 { if frequency == 0 { return 0; } (tick / frequency) * 24_000_000 + (tick % frequency) * 24_000_000 / frequency } + +/// Adopts the running firmware's UAT on `soc`: the `mmu::Uat` of the admitted reserved regions +/// (ttbs, pagetables, handoff, shared-l2) and the PPL exchange with its startup epoch, as the +/// G15 backend does it on 7.2 (`g15::uat`). +fn adopt_uat(drm: &driver::AsahiDrmDevice, soc: &'static crate::m3::m3_soc::Soc, + resources: &crate::m3::agx_resources::Resources) -> Result<(mmu::Uat, crate::g15::uat::Window)> { + let cfg = soc.hwcfg.ok_or(ENODEV)?; + let [ttbs, tables, handoff, shared_l2, ..] = resources.regions; + let range = |r: crate::m3::agx_resources::Region| r.base..r.base + r.size; + let geometry = mmu::UatGeometry { ias: cfg.uat_ias.into(), oas: cfg.uat_oas }; + let regions = mmu::AdoptedRegions { handoff: range(handoff), ttbs: range(ttbs), kernel_root: tables.base }; + let uat = mmu::Uat::new_adopted(drm, geometry, ®ions)?; + let mut window = crate::g15::uat::Window::from_regions(handoff.base, ttbs.base, tables.base, shared_l2.base)?; + window.bootstrap(drm.as_ref().as_ref(), &uat)?; + // 7.1.12 cleaned every reserved-table write to the point of coherency; these regions may be + // reserved System RAM mapped cacheable. + window.clean_tables(); + Ok((uat, window)) +} diff --git a/drivers/gpu/drm/asahi/m3/m3_soc.rs b/drivers/gpu/drm/asahi/m3/m3_soc.rs index f19717d536f723..7be649415cccce 100644 --- a/drivers/gpu/drm/asahi/m3/m3_soc.rs +++ b/drivers/gpu/drm/asahi/m3/m3_soc.rs @@ -2,48 +2,230 @@ //! The per-SoC facts of the M3 runtime backend. //! -//! Each table describes the configuration of one GPU. This compile-only subset holds only the -//! facts the M3 boundary carries; the register windows, memory layout, firmware images and -//! admission of the full table stay with the M3 runtime port. Nothing here is read by a probe. - -use super::m3_compute_layout::RegisterSet; - -/// The HwDataB words of the runtime backend's InitData that are neither its GPU VA layout nor -/// device-tree values: configuration words where the manager layout keeps 0 or its validated -/// values (including the A/B pair +0xa98/+0xb40 at 0), and the two unit masks. They are values -/// of one SoC's runtime InitData (`m3_soc::Soc::hwdata_b`), not derived from the device tree. -#[derive(Debug)] -pub(crate) struct G15RuntimeHwDataB { - pub(crate) unk_454: u32, - pub(crate) unk_464: u32, - pub(crate) unk_a7c: u64, - pub(crate) unk_a98: u32, - pub(crate) unk_abc: u64, - pub(crate) unk_ae4: u32, - pub(crate) unk_b20: u32, - pub(crate) unk_b24: u64, - /// +0xb40, the "UAT enabled" word of the manager layout. - pub(crate) unk_554: u32, - pub(crate) unk_17b8: u32, - /// Unit enable masks, +0x17c0/+0x17c8. - pub(crate) unit_mask_a: u64, - pub(crate) unit_mask_b: u32, - pub(crate) unk_1808: u32, - pub(crate) unk_1818: u32, +//! Each table describes the register windows, memory layout and configuration of one GPU. +//! Missing configuration is `None`; outstanding runtime requirements are listed in `unported`. +//! Admission rejects either before reading firmware, mapping GPU registers or starting the +//! GPU coprocessor ([`Soc::require_complete`]). + +use kernel::{c_str, device, prelude::*}; + +use crate::{ + m3::hw, + m3::initdata::G15RuntimeHwDataB, + m3::m3_compute_layout::RegisterSet, + m3::m3_board::KnownImage, + m3::m3_firmware::Layout, + m3::m3_init_storage::IoMap, // +}; + +/// The SGX identification words the runtime admits, each compared under its mask. +pub(crate) struct IdWords { + /// SGX+0xd04000: family [31:24], variant [23:16], revision [15:8]. + pub(crate) version: u32, + pub(crate) version_mask: u32, + /// SGX+0xd04010: dies [19:16], clusters per die [15:8]. + pub(crate) counts: u32, + pub(crate) counts_mask: u32, } +/// The register windows the runtime admits, as CPU physical addresses (device-tree `reg` values +/// translated through `arm-io` `ranges`). +pub(crate) struct Windows { + /// Base of the GPU coprocessor window (`asc`, ADT `gfx-asc` reg[0]). + pub(crate) asc: u64, + /// Base of the GPU window (`sgx`, ADT `sgx` reg[0]). + pub(crate) sgx: u64, + /// Base of the GPU coprocessor mailbox, a 16 KiB window inside `asc`. + pub(crate) mailbox: u64, +} + +/// The performance-state table the runtime accepts: checked on the generated InitData before +/// any GPU register is touched, and again on the uploaded HwDataB. +#[derive(Copy, Clone, Debug)] +pub(crate) enum PstateTable { + /// A fixed table: `states` states above the off state, the highest at `top_mhz` MHz. + Fixed { states: u32, top_mhz: u32 }, + /// The table of the device tree's operating points: one state per distinct voltage above + /// the off state, the highest at the highest frequency + /// (`t8122_admission::opp_table_shape`). + DeviceTree, +} + +/// Optional userspace features advertised for a SoC (`DRM_ASAHI_GET_PARAMS`). The runtime +/// implements them the same way on every SoC; a SoC advertises one once it has been validated +/// there. Soft faults are never advertised: the runtime does not apply `asahi.fault_control`. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Features { + /// `DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY`. + pub(crate) compute_wide_visibility: bool, + /// `DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY`, while `asahi.m3_early_tiling` is set. + pub(crate) fragment_dependency: bool, +} + +/// The MTR temperature-sensor masks of the runtime's HwDataA, where the SoC table sets them +/// instead of the hardware configuration. +#[derive(Copy, Clone, Debug)] +pub(crate) struct MtrMasks { + /// The fast-die sensor mask, in both HwDataA copies (+0x8ac and +0x1288). + pub(crate) fast_die: u64, + /// The mask the firmware matches an MTR alarm against (HwDataA +0x1a98). An alarm from a + /// sensor outside it, or any alarm while it is 0, is fatal to the firmware. + pub(crate) alarm: u64, +} + +/// Where the boot loader exports a SoC's decoded GPU leakage fuse values, in `/chosen`, and the +/// boot loader switch that makes them this boot's leakage. +#[derive(Copy, Clone, Debug)] +pub(crate) struct LeakFuse { + /// Two cells, value 1 then value 2, encoded as `apple,core-leak-coef` is. + pub(crate) values: &'static CStr, + /// The switch: the boot loader uses the values exactly when it holds the string "1" (the + /// two bytes `1\0`), the form it copies from its configuration line. + pub(crate) switch: &'static CStr, +} + +/// Where the runtime places the HwData object (`m3_init_storage::HARDWARE_DATA`), when it is not +/// the fixed allocation's address, and how the firmware maps it. +#[derive(Copy, Clone, Debug)] +pub(crate) struct HwDataObject { + /// Firmware VA of the object. + pub(crate) address: u64, + /// Map the object cacheable for the firmware (its atomic updates need cacheable memory) + /// instead of uncached. + pub(crate) cached: bool, +} + +/// One firmware IO mapping of the runtime's InitData: HwDataB slot, physical address, total +/// size, element size, writable. +pub(crate) type IoMapping = (usize, u64, u32, u32, bool); + +/// The M3 runtime's facts for one SoC. pub(crate) struct Soc { /// SoC name, for logs. pub(crate) name: &'static str, + /// GPU name, for logs. + pub(crate) gpu_name: &'static str, + pub(crate) chip_id: u32, + pub(crate) gpu_variant: hw::GpuVariant, + pub(crate) gpu_revision: hw::GpuRevision, + pub(crate) gpu_revision_id: hw::GpuRevisionID, + /// Root node compatible of the SoC. + pub(crate) board: &'static str, + /// GPU node compatible. + pub(crate) gpu: &'static str, + /// Boards the runtime starts on by default and registers a render node on. On a T8122 it + /// starts on no other board, whatever `asahi.m3_backend` says (`m3_params::t8122_backend`). + pub(crate) validated_boards: &'static [&'static [u8]], + /// GPU clusters, and core slots per cluster. + pub(crate) clusters: u32, + pub(crate) cores_per_cluster: u32, + /// Accepted compatible lists of the GPU coprocessor mailbox. + pub(crate) mailbox_compatibles: &'static [&'static [u8]], + /// The mailbox interrupts: send-empty, send-not-empty, recv-empty, recv-not-empty, as the + /// mailbox node's `interrupts` cells (three per interrupt, or four with the die cell of a + /// multi-die interrupt controller). + pub(crate) mailbox_interrupts: &'static [u32], + /// The register windows of the GPU node and its mailbox. + pub(crate) windows: Windows, + /// The identification words the runtime admits. + pub(crate) id: IdWords, + /// GPU firmware images the runtime can identify. + pub(crate) images: &'static [KnownImage], + /// The layout of the loaded firmware the runtime accepts. + pub(crate) firmware: Option<&'static Layout>, + /// The hardware configuration of the InitData. + pub(crate) hwcfg: Option<&'static hw::HwConfig>, + /// The firmware IO mappings of the InitData. + pub(crate) io_mappings: Option<&'static [IoMapping]>, + /// The runtime's mapping of each of those IO mappings: the CPU physical block and the + /// firmware VA it is mapped at. One per entry of `io_mappings`, covering it. + pub(crate) iomaps: Option<&'static [IoMap]>, + /// The performance-state table the runtime accepts. + pub(crate) pstates: PstateTable, + /// The SGX write (offset, value) made after the identity checks, before the firmware starts. + pub(crate) sgx_setup: Option<(usize, u32)>, /// The runtime InitData's HwDataB configuration words and unit masks. pub(crate) hwdata_b: Option<&'static G15RuntimeHwDataB>, + /// The optional userspace features advertised. + pub(crate) features: Features, + /// Whether the power-management coefficients, the leakage coefficients and the operating + /// points are this machine's, added by the boot loader from its ADT, rather than static + /// device-tree values. Admission then requires every one of them (see + /// `m3_board::check_boot_loader_power`), with no driver default standing in except the + /// idle-off standby timer, which an ADT may lack. + pub(crate) power_from_boot_loader: bool, + /// Code the runtime runs after admission that still holds another SoC's values. + pub(crate) unported: &'static [&'static str], + /// The GPU registers read after every job by default (`asahi.m3_retire_mmio` bits). + pub(crate) retire_mmio: u64, + /// The MTR sensor masks, when they are not the hardware configuration's. + pub(crate) mtr_masks: Option, + /// The HwData object's placement and mapping, when they are not the fixed allocation's. + pub(crate) hwdata_object: Option, /// The register lists the GPU's firmware expects in compute and render commands. pub(crate) registers: RegisterSet, + /// Globals +0x7d0, which gates the firmware's frequency-feedback cap: the firmware measures + /// the GPU clock against the requested performance state and, while this is 1, caps the + /// state on a shortfall (`asahi.m3_ut_engagement` overrides it). + pub(crate) ut_engagement: u32, + /// The highest GPU power target, in mW, while the boot loader's power model is a stand-in: + /// the operating points' powers are scaled down together so that the highest is this. + pub(crate) power_target_cap_mw: Option, /// Words of the generated HwData object (offset into the object, value) that this SoC's /// firmware takes with other values than the shared builder writes. pub(crate) hwdata_words: &'static [(usize, u32)], /// The same for the Globals object. pub(crate) globals_words: &'static [(usize, u32)], + /// The boot loader's decoded GPU leakage fuse, when it exports one for this SoC. + pub(crate) leak_fuse: Option, + /// Whether each operating point may give each cluster its own voltage (the per-cluster, + /// binned voltage tables of T6031). Otherwise every cluster of a point must have the same. + pub(crate) per_cluster_voltages: bool, +} + +impl Soc { + /// Reject incomplete configuration before accessing the GPU. + pub(crate) fn require_complete(&self, dev: &device::Device) -> Result { + let mut missing = 0u32; + let mut note = |what: &str| { + dev_info!(dev, "M3 {}: missing {}\n", self.gpu_name, what); + missing += 1; + }; + if !self.images.iter().any(|image| image.initdata_magic.is_some()) { + note("the identity of the loaded GPU firmware image and its InitData version"); + } + if self.firmware.is_none() { + note("the layout of the loaded GPU firmware (segment sizes and VAs)"); + } + if self.hwcfg.is_none() { + note("the hardware configuration of the InitData (HwConfig)"); + } + if self.io_mappings.is_none() { + note("the firmware IO mappings of the InitData"); + } + if self.iomaps.is_none() { + note("the runtime's mapping of the firmware IO mappings (IO maps)"); + } + if self.sgx_setup.is_none() { + note("the SGX setup write made before the firmware starts"); + } + if self.hwdata_b.is_none() { + note("the runtime HwDataB configuration words and unit masks"); + } + for &what in self.unported { + note(what); + } + if missing == 0 { + return Ok(()); + } + dev_err!( + dev, + "M3 {}: {} configuration requirements missing; GPU startup disabled\n", + self.gpu_name, + missing + ); + Err(ENODEV) + } } /// The HwDataB configuration words and unit masks of the T6030 runtime InitData, as checked on @@ -77,24 +259,115 @@ pub(crate) static T8122_HWDATA_B: G15RuntimeHwDataB = G15RuntimeHwDataB { }; /// Every SoC the M3 runtime has a table for. -pub(crate) static SOCS: [&Soc; 2] = [&T6030, &T8122]; - -/// T6030 (M3 Pro, G15S): one die, two clusters of ten core slots. -pub(crate) static T6030: Soc = Soc { - name: "T6030", - hwdata_b: Some(&T6030_HWDATA_B), - registers: RegisterSet::G15S, - hwdata_words: &[], - globals_words: &[], -}; +pub(crate) static SOCS: [&Soc; 1] = [&T8122]; + +/// The table of the SoC with chip id `chip_id`, if the M3 runtime has one. +pub(crate) fn by_chip(chip_id: u32) -> Option<&'static Soc> { + SOCS.iter().copied().find(|soc| soc.chip_id == chip_id) +} /// T8122 (M3, G15G): one die, one cluster of ten core slots (eight or ten of them active). +/// +/// The compatibles, the mailbox and its interrupts are the T8122 device tree's. The identity is +/// the one the T8122 identity gate admits (`t8122_admission`: family 7, variant 2, revision 0x20, +/// core slots in the first core-mask word only), with the die count of the AGX3 identification +/// table (`hw::agx3::T8122`). The hardware configuration is `hw::t8122`; power configuration +/// comes from the boot loader. The rest of the runtime configuration is complete, but until the +/// runtime has passed on an M3 MacBook Air the SoC is refused, before the GPU is accessed, unless +/// `asahi.t8122_start=1` arms the start (`t8122_start`), which uses this table's values and adds a +/// performance-state cap. pub(crate) static T8122: Soc = Soc { name: "T8122", + gpu_name: "G15G", + chip_id: 0x8122, + gpu_variant: hw::GpuVariant::G, + gpu_revision: hw::GpuRevision::C0, + gpu_revision_id: hw::GpuRevisionID::C0, + board: "apple,t8122", + gpu: "apple,agx-t8122", + // The M3 MacBook Airs (13" J613, 15" J615). Allowed, not validated: the runtime has not run + // on either, and starts only behind a boot loader that hands the GPU over. + validated_boards: &[b"apple,j613", b"apple,j615"], + clusters: 1, + cores_per_cluster: 10, + mailbox_compatibles: &[ + b"apple,t8122-agx-asc-mailbox\0", + b"apple,t8122-asc-mailbox\0apple,asc-mailbox-v4\0", + ], + mailbox_interrupts: &[0, 723, 4, 0, 724, 4, 0, 725, 4, 0, 726, 4], + // The J613 ADT: sgx reg[0] (child 0x80000000 + arm-io 0x210000000) and gfx-asc reg[0]; the + // mailbox at asc + 0x8000 (t8122-gpu.dtsi). The same addresses as on T6030. + windows: Windows { + asc: 0x2_9240_0000, + sgx: 0x2_9000_0000, + mailbox: 0x2_9240_8000, + }, + id: IdWords { + version: 0x0702_2000, + version_mask: 0xffff_ff00, + // One die, one cluster per die; the other fields are not known. + counts: 0x0001_0100, + counts_mask: 0x000f_ff00, + }, + images: &crate::m3::m3_board::KNOWN_IMAGES_T8122, + firmware: Some(&crate::m3::m3_firmware::T8122_LAYOUT), + hwcfg: Some(&hw::t8122::HWCONFIG_T8122), + io_mappings: Some(&crate::m3::m3_adt_config::T8122_IO_MAPPINGS), + iomaps: Some(&crate::m3::m3_adt_config::T8122_IOMAPS), + // The boot loader's ladder from this machine's ADT (J613: eight voltages, up to 1338 MHz). + pstates: PstateTable::DeviceTree, + // The same SGX setup write as on T6030. + sgx_setup: Some((0xd14000, 0x70001)), hwdata_b: Some(&T8122_HWDATA_B), + // Neither is validated on G15G yet; userspace keeps its default ordering and visibility. + features: Features { + compute_wide_visibility: false, + fragment_dependency: false, + }, + power_from_boot_loader: true, + unported: &[ + "a pass on an M3 MacBook Air: until then the GPU starts only with asahi.t8122_start=1", + ], + // The performance state only. On the single-cluster G15G, reading the engine-busy or the + // fault-bank registers after every job hangs the SoC within seconds at about 1000 jobs/s; + // the performance-state read alone does not, and a per-frame readback saw no job retired + // before its writes were visible. + retire_mmio: 4, + // The fast-die controller's sensors (0x4248, both copies), and every sensor the MTR block + // enables as the alarm mask (sensors 3, 6, 8, 9, 11 and 14: 0x4b48, the upper half of the + // block's configuration word 0x4b480003). With the second fast-die copy at 0, or an alarm + // mask without sensor 8, the firmware stopped on its first MTR alarm. + mtr_masks: Some(MtrMasks { + fast_die: 0x4248, + alarm: 0x4b48, + }), + // The firmware's MTR alarm handler does a 64-bit atomic update of HwDataA +0x4350. The fixed + // allocation ends the 0x8a04-byte object at its page end, leaving HwDataA (+0x4580) only + // 4-byte aligned, and the update took an alignment fault. Starting the object 0x8a80 bytes + // before the same page end puts HwDataA on a 16-byte boundary. Aligned, the update then + // faulted as an unsupported atomic on the uncached mapping: map it cacheable. + hwdata_object: Some(HwDataObject { + address: T8122_HWDATA_ADDRESS, + cached: true, + }), registers: RegisterSet::G15G, + // On a T8122 with the 14.8.3 system firmware the shader clock runs at 3/4 of every requested + // state, so with the cap engaged the GPU stays at state 2 (462 MHz effective) under any load. + // With 0 it reaches the requested states (state 8: about 1000 MHz effective). + ut_engagement: 0, + // The boot loader's T8122 power model is a stand-in that overstates the power (its highest + // operating point is about 30 W). Until each Mac's fused leakage gives a real one, scale it + // to a 22 W target, about the GPU power budget of an M3 MacBook Air. + power_target_cap_mw: Some(22_000), hwdata_words: &T8122_HWDATA_WORDS, globals_words: &T8122_GLOBALS_WORDS, + // Exported on a J613 on every boot; used only with the boot loader's switch, which also makes + // value 1 its apple,core-leak-coef and the base of its operating points' powers. + leak_fuse: Some(LeakFuse { + values: c_str!("asahi,t8122-gpu-leak-fuse"), + switch: c_str!("asahi,t8122-gpu-fuse-leakage"), + }), + per_cluster_voltages: false, }; /// The T8122 HwData words that differ from the shared builder's (offsets into the object; @@ -104,7 +377,7 @@ pub(crate) static T8122: Soc = Soc { /// +0x1804, +0x1814 and +0x1860 and all-ones at +0x1848); /// - HwDataA +0x11e0: 30 (T6030: 40), a word of the shader-engine controller block; /// - HwDataA +0x1290: 125, and HwDataA +0x424c: 24000000, the 24 MHz reference clock. -static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ +pub(crate) static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ (0xa30, 0), (0xa34, 4), (0x17e0, 1), @@ -126,4 +399,18 @@ static T8122_HWDATA_WORDS: [(usize, u32); 17] = [ /// The T8122 Globals words that differ from the shared builder's: +0x9bc, the CDM backoff /// timeout (4) with the three bytes after it 0 (T6030: 1, 0, 0). -static T8122_GLOBALS_WORDS: [(usize, u32); 1] = [(0x9bc, 4)]; +pub(crate) static T8122_GLOBALS_WORDS: [(usize, u32); 1] = [(0x9bc, 4)]; + +/// The T8122 HwData object's firmware VA: 0x8a80 bytes before the end of the fixed allocation's +/// last page. +pub(crate) const T8122_HWDATA_ADDRESS: u64 = 0xffff_fc20_4070_4000 - 0x8a80; + +// The fixed HwData allocation (`m3_init_storage::allocation`): 0x8a04 bytes ending at the page +// end 0xfffffc2040704000. The moved object stays inside the same pages, and HwDataA is 16-byte +// aligned. +const _: () = { + let (fixed, size) = (0xffff_fc20_406f_b5fc_u64, 0x8a04_u64); + assert!(T8122_HWDATA_ADDRESS & !0x3fff == fixed & !0x3fff); + assert!(T8122_HWDATA_ADDRESS + size <= (fixed + size + 0x3fff) & !0x3fff); + assert!((T8122_HWDATA_ADDRESS + crate::m3::m3_adt_config::HWDATA_A as u64) % 16 == 0); +}; diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 9bd08d0257b0c9..326e5148083820 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Bounded M3 compute and graphics submissions through the common DRM scheduler/syncobjs. use kernel::{c_str,dma_fence::*,drm::sched,prelude::*,sync::{Arc,Mutex},new_mutex,xarray}; -use crate::{file,agx_uapi,mmu,queue,m3_drm::Shared}; +use crate::{file,m3::agx_uapi,mmu,queue,m3::m3_drm::Shared}; use core::sync::atomic::{AtomicBool, Ordering}; #[derive(Default)] pub(crate) struct Completion; @@ -32,14 +32,14 @@ impl Destination { #[derive(Copy,Clone)] pub(crate) enum Command { - Compute(crate::agx_compute::Control), + Compute(crate::m3::agx_compute::Control), Render{command:agx_uapi::UapiRenderCommand,usc:u64}, } pub(crate) struct Packet { pub(crate) vm:mmu::Vm, pub(crate) commands:KVec, timestamps:KVec<[Option;4]>, - pub(crate) attachments:KVec, + pub(crate) attachments:KVec, pub(crate) wide_visibility:KVec, completion:UserFence, finish_claimed: AtomicBool, @@ -47,7 +47,7 @@ pub(crate) struct Packet { pub(crate) urgency:u8, /// Who submitted the packet, for the failure log only. pub(crate) owner:Owner, - vm_job:Pin>>>, + vm_job:Pin>>>, } /// The process and thread that submitted a packet, recorded in the submit ioctl. Only logged: /// a renamed task leaves a stale name, which is harmless. @@ -57,8 +57,8 @@ impl Owner { fn current()->Self { let task=kernel::current!(); let leader=task.group_leader(); - Self {pid:leader.pid(),process:crate::m3_client::task_comm(leader), - thread:crate::m3_client::task_comm(task)} + Self {pid:leader.pid(),process:crate::m3::m3_client::task_comm(leader), + thread:crate::m3::m3_client::task_comm(task)} } fn name(comm:&[u8;16])->&str { let len=comm.iter().position(|&b|b==0).unwrap_or(comm.len()); @@ -84,18 +84,18 @@ impl Packet { // Claim error/signal ownership once; this flag is NOT proof that the // fence is signaled or that hardware DMA has stopped. if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } - if let Err(e)=result {if let Ok(status) = self.vm.agx_status() { status.record(e.to_errno()); }self.completion.set_error(e);} + if let Err(e)=result {if let Some(status) = self.vm.status() { status.record(e); }self.completion.set_error(e);} // A scheduler job/fence and the runtime's last packet can outlive // hardware retirement. They must not keep the VM's active-job count // nonzero: GEM closes otherwise accumulate across completed frames. // On failure the runtime quarantines this guard with the packet until // firmware has provably stopped; signalling an error is not retirement. - crate::agx_memory::publish(); + crate::m3::agx_memory::publish(); let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); - let before=if profile {crate::m3_runtime::physical_counter()} else {0}; + let before=if profile {crate::m3::m3_runtime::physical_counter()} else {0}; self.completion.signal(); if profile { - let after=crate::m3_runtime::physical_counter(); + let after=crate::m3::m3_runtime::physical_counter(); let status=match result {Ok(())=>0,Err(e)=>e.to_errno()}; pr_info!("M3_PACKET_FENCE vm={} packet={:#x} commands={} signal_before_tick={} signal_after_tick={} status={} counter_hz=24000000\n", self.vm.id(),self.profile_id(),self.commands.len(),before,after,status); @@ -113,7 +113,7 @@ impl sched::JobImpl for Job { fn run(job:&mut Self)->Result> { // Queue the packet and return its fence; the completion worker publishes and retires // it, so the scheduler can hand over further ready jobs meanwhile. - if let Err(e)=crate::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { + if let Err(e)=crate::m3::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { pr_err!("M3 scheduler: submission failed {:?}\n",e); // submit() failed before this packet could be published. job.packet.release_guard();job.packet.finish(Err(e)); @@ -122,9 +122,9 @@ impl sched::JobImpl for Job { } fn timed_out(job:&mut JobTimeout,_finished:bool)->sched::Status { // Retain the publishing branch's explicit diagnostic override. - if !crate::m3_params::timeout_nohang() { + if !crate::m3::m3_params::timeout_nohang() { if let Some(r)=Option::as_mut(&mut *job.shared.lock()) {r.health().mark_failed();} - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(ETIMEDOUT));return sched::Status::NoDevice; } let guard=job.shared.lock(); @@ -138,12 +138,12 @@ impl sched::JobImpl for Job { // device loss or corrupting an already signaled fence. if runtime.health().healthy() { return sched::Status::NoHang; } drop(guard); - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(EIO)); sched::Status::NoDevice } fn cancel(job:&mut Self) { - crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); + crate::m3::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(ECANCELED)); } } @@ -178,9 +178,9 @@ impl Queue { } } impl queue::Queue for Queue { - fn submit(&mut self,_id:u64,mut syncs:KVec,in_sync_count:usize, + fn submit(&mut self,_reg_data:&crate::driver::AsahiRegData<'_>,_id:u64,mut syncs:KVec,in_sync_count:usize, raw:&[u8],objects:Pin<&xarray::XArray>>)->Result { - let vm_job=self.vm.retain_m3_job()?; + let vm_job=self.vm.retain_job()?; let mut parser=agx_uapi::UapiCommandParser::new(raw);let mut commands=KVec::new();let mut timestamps=KVec::new();let mut flushes=KVec::new();let mut visibility=KVec::new(); while let Some(command)=parser.next_hardware().map_err(|_|EINVAL)? { if let agx_uapi::ParsedHardwareCommand::Render{payload,vertex_attachments,fragment_attachments,..}=command { @@ -195,7 +195,7 @@ impl queue::Queue for Queue { Destination::resolve(objects,payload.vertex_timestamps.end)?, Destination::resolve(objects,payload.fragment_timestamps.start)?, Destination::resolve(objects,payload.fragment_timestamps.end)?],GFP_KERNEL)?; - flushes.push(crate::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; + flushes.push(crate::m3::agx_attachments::Attachments::EMPTY,GFP_KERNEL)?; visibility.push(false,GFP_KERNEL)?; continue; } @@ -217,10 +217,10 @@ impl queue::Queue for Queue { pr_err!("M3 compute translation rejected {:?}: stream={:#x}..{:#x} helper={:?}\n",e,payload.control_stream_base,payload.control_stream_end,payload.helper); EINVAL })?; - crate::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; + crate::m3::m3_client::check_compute_launches(&self.vm,&AddressSpace(&self.vm),c.control_stream_base,c.control_stream_end)?; timestamps.push([Destination::resolve(objects,payload.timestamps.start)?, Destination::resolve(objects,payload.timestamps.end)?,None,None],GFP_KERNEL)?; - commands.push(Command::Compute(crate::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; + commands.push(Command::Compute(crate::m3::agx_compute::Control{base:c.control_stream_base,end:c.control_stream_end,usc_base:self.usc}),GFP_KERNEL)?; } parser.finish().map_err(|_|EINVAL)?; let packet=Arc::new(Packet{vm:self.vm.clone(),commands,timestamps,attachments:flushes,wide_visibility:visibility, @@ -238,8 +238,8 @@ impl queue::Queue for Queue { } fn encode_attachments(vm: &mmu::Vm, list: &agx_uapi::UapiAttachmentList) - -> Result { - use crate::agx_attachments::{Attachments, CAPACITY}; + -> Result { + use crate::m3::agx_attachments::{Attachments, CAPACITY}; use agx_uapi::{GpuAddressSpace, GpuAccess}; if list.as_slice().len()>CAPACITY {return Err(E2BIG);} let mut ranges = [(0, 0); CAPACITY]; @@ -262,7 +262,7 @@ fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpac if r.sample_size % 8 != 0 {return Err(ENOTSUPP);} let empty = agx_uapi::UapiHelperProgram { binary:0, config:0, data:0 }; // Early tiling (render flag bit 5) only where it is advertised. - let stage_scope=if early_tiling && *crate::module_parameters::m3_early_tiling.value()!=0 {1<<5} else {0}; + let stage_scope=if early_tiling && crate::m3::m3_params::early_tiling() {1<<5} else {0}; if r.flags & !(2|16|(1<<18)|stage_scope) != 0 || !matches!(r.samples, 1 | 2 | 4) || r.sampler_count != 0 || r.sampler_heap != 0 || r.vertex_helper != empty || r.fragment_helper != empty @@ -279,7 +279,7 @@ fn validate_render(r: agx_uapi::UapiRenderCommand, usc: u64, space: &AddressSpac || u32::from(r.sample_size)*u32::from(r.utile_width)*u32::from(r.utile_height)*u32::from(r.samples)>32768 { return Err(EINVAL); } - crate::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; + crate::m3::agx_render_state::compact(r.vdm_base).map_err(|_| EINVAL)?; if r.vdm_base & 3 != 0 || !space.covers(r.vdm_base,4,GpuAccess::Read) || r.scissor_base == 0 || r.scissor_base & 7 != 0 || !space.covers(r.scissor_base,8,GpuAccess::Read) { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/m3/m3_thermal.rs b/drivers/gpu/drm/asahi/m3/m3_thermal.rs index 59a72187c96990..e30f8fe011332d 100644 --- a/drivers/gpu/drm/asahi/m3/m3_thermal.rs +++ b/drivers/gpu/drm/asahi/m3/m3_thermal.rs @@ -47,7 +47,7 @@ use core::sync::atomic::{AtomicBool, Ordering}; use kernel::{ - bindings, c_str, device, + c_str, device, error::to_result, new_spinlock, prelude::*, @@ -58,9 +58,9 @@ use kernel::{ }; use crate::{ - m3_adt_config::PstatePolicy, - m3_params::{self, ThermalMode}, - m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, + m3::m3_adt_config::PstatePolicy, + m3::m3_params::{self, ThermalMode}, + m3::m3_thermal_policy::{Change, Policy, Reading, Source, Why, S}, }; /// The thermal zones read, each the hottest of a set of SoC die sensors: the SMC's die sensors @@ -78,7 +78,7 @@ fn required_zones() -> u32 { let Ok(compatible) = root.get_property::>(c_str!("compatible")) else { return J516_REQUIRED_ZONES; }; - crate::m3_thermal_policy::required_zones(&compatible) + crate::m3::m3_thermal_policy::required_zones(&compatible) } /// Sensor polling period while the GPU is busy and while it is idle, in ms. @@ -122,13 +122,18 @@ impl_has_delayed_work! { impl HasDelayedWork for Sensor { self.work } } +extern "C" { + // include/linux/thermal.h; not in 7.2's generated bindings. + fn thermal_zone_get_temp_by_name(name: *const core::ffi::c_char, temp: *mut i32) -> core::ffi::c_int; +} + /// One reading of the zone `name`, in millidegrees Celsius; ENODEV if there is no such zone. fn read_zone(name: &CStr) -> Result { let mut temp: i32 = 0; // SAFETY: name is NUL-terminated and temp is a valid output pointer. The // thermal core acquires a device reference under its registry lock and // keeps it through sampling, so zone unregistration cannot free it here. - to_result(unsafe { bindings::thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; + to_result(unsafe { thermal_zone_get_temp_by_name(name.as_char_ptr(), &mut temp) })?; Ok(temp) } diff --git a/drivers/gpu/drm/asahi/m3/mod.rs b/drivers/gpu/drm/asahi/m3/mod.rs index 3d900a80fef382..1d10657c02f662 100644 --- a/drivers/gpu/drm/asahi/m3/mod.rs +++ b/drivers/gpu/drm/asahi/m3/mod.rs @@ -43,6 +43,96 @@ pub(crate) fn refuse_probe(pdev: &platform::Device>, chip: Chip) -> Err ENODEV } +/// Whether the M3 runtime may start on this T8122 (M3, G15G), whose GPU the boot loader handed +/// over: its identity gate passes (only DT properties are read) and the backend selection allows +/// the board. The runtime then admits the boot loader's description, and refuses the SoC unless +/// `asahi.t8122_start=1` arms the start ([`t8122_start`]). +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) fn t8122_admitted(pdev: &platform::Device>) -> Result<&'static m3_soc::Soc> { + let identity = t8122_admission::read_identity(pdev).map_err(|error| { + dev_info!( + pdev.as_ref(), + "G15G: T8122 identity or ABI rejected ({:?}); startup refused\n", + error + ); + ENODEV + })?; + dev_info!( + pdev.as_ref(), + "G15G: T8122 revision 2.0 identity admitted ({} active / {} slots)\n", + identity.active_cores, + identity.core_slots + ); + if !m3_params::t8122_backend(pdev) { + return Err(ENODEV); + } + Ok(&m3_soc::T8122) +} + +// The T8122 runtime, ported from the 7.1.12 release-20 kernel (c46d7dcdb871). +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_attachments; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_compute; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_memory; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_resources; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_status; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod agx_uapi; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod g15_raw; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod hw; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod identity; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod initdata; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_adt_config; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_board; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_client; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_completion; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_compute; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_config; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_device; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) mod m3_drm; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_firmware; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_memory; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_params; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_pass; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_pool; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_render; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_resources; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_rtkit; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_runtime; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_submit; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3_thermal; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod t8122_knobs; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod t8122_start; + mod m3_compute_layout; mod m3_compute_sequence; @@ -103,6 +193,8 @@ mod m3_fragment_command; mod t8122_admission; -mod m3_soc; +// The runtime's SoC tables; they name the runtime's own modules. +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +pub(crate) mod m3_soc; mod t8122_hwdata; diff --git a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs index f9c3ff08006d5b..b48e561f677ef1 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_knobs.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_knobs.rs @@ -172,7 +172,7 @@ fn given(param: u64, default: u64) -> Option { /// Resolve the parameters. The first parameter with a value it does not accept is the refusal. pub(crate) fn resolve(raw: &Raw) -> Result { let refuse = |name, accepts| Refusal { name, accepts }; - let initdata_version = given(raw.initdata_version, crate::m3_firmware::G15_V14_8_3_INITDATA) + let initdata_version = given(raw.initdata_version, crate::m3::m3_firmware::G15_V14_8_3_INITDATA) .filter(|v| *v != 0) .ok_or(refuse("t8122_initdata_version", "a nonzero number, at most 0xfffffffffffffffd"))?; let fender = match given(raw.fender, FENDER_RULE as u64) { diff --git a/drivers/gpu/drm/asahi/m3/t8122_start.rs b/drivers/gpu/drm/asahi/m3/t8122_start.rs index cf0ee658e8da50..ca9f21b56dbb66 100644 --- a/drivers/gpu/drm/asahi/m3/t8122_start.rs +++ b/drivers/gpu/drm/asahi/m3/t8122_start.rs @@ -27,13 +27,13 @@ use core::mem::offset_of; use kernel::{device, prelude::*}; use crate::{ - fw::initdata::raw, - initdata::G15RuntimeHwDataB, - m3_adt_config::T8122_IO_MAPPINGS, - m3_init_storage::{self as storage, IoMap}, - m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, + m3::g15_raw::raw, + m3::initdata::G15RuntimeHwDataB, + m3::m3_adt_config::T8122_IO_MAPPINGS, + m3::m3_init_storage::{self as storage, IoMap}, + m3::m3_soc::{IoMapping, Soc, T8122_HWDATA_B}, pgtable::{prot, Prot}, - t8122_knobs::{ + m3::t8122_knobs::{ self as knobs, ClockGen, Start, Values, CLOCK_GEN_E1C, CLOCK_GEN_E5C, FENDER_ADT, FENDER_RULE, }, @@ -140,7 +140,7 @@ const _: () = { assert!(offset_of!(raw::HwDataBG15V14_8_3, unit_mask_b) == 0x17c8); // An armed start skips `require_complete`. The only `unported` item of the T8122 table is the // opt-in itself; if another is added, this fails the build until the start accounts for it. - assert!(crate::m3_soc::T8122.unported.len() == 1); + assert!(crate::m3::m3_soc::T8122.unported.len() == 1); }; /// The values of one armed boot. @@ -149,7 +149,7 @@ pub(crate) struct Experiment(Values); /// Whether `soc` is the T8122 table. pub(crate) fn is_t8122(soc: &Soc) -> bool { - core::ptr::eq(soc, &crate::m3_soc::T8122) + core::ptr::eq(soc, &crate::m3::m3_soc::T8122) } impl Experiment { @@ -182,7 +182,7 @@ impl Experiment { /// they are the T8122 table's own IO maps. pub(crate) fn iomaps(&self) -> &'static [IoMap] { if self.0.fender == FENDER_RULE && self.0.clock_gen == ClockGen::At(CLOCK_GEN_E1C) { - return &crate::m3_adt_config::T8122_IOMAPS; + return &crate::m3::m3_adt_config::T8122_IOMAPS; } &IOMAPS[self.variant()][..self.mapping_count()] } @@ -245,7 +245,7 @@ pub(crate) fn arm(dev: &device::Device, soc: &Soc) -> Result> if !is_t8122(soc) { return Ok(None); } - let raw = crate::m3_params::t8122_params(); + let raw = crate::m3::m3_params::t8122_params(); match knobs::start(&raw) { Start::On => {} Start::Off => { @@ -354,7 +354,7 @@ pub(crate) fn refused(dev: &device::Device, experiment: Option<&Experiment>, sta /// Log, on a T8122 armed with `asahi.t8122_start=1`, that resource admission refused it. pub(crate) fn not_admitted(dev: &device::Device, soc: &Soc) { - if is_t8122(soc) && knobs::start(&crate::m3_params::t8122_params()) == Start::On { + if is_t8122(soc) && knobs::start(&crate::m3::m3_params::t8122_params()) == Start::On { dev_err!( dev, "M3 G15G start: not armed: resource admission refused the GPU (see the lines above); the boot loader did not hand it over completely\n" diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index e868e30baca4c9..dc828a460202d5 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -117,6 +117,8 @@ mod bind; mod context; mod shared; mod lifetime; +#[cfg(CONFIG_DRM_ASAHI_M3_T8122)] +mod m3; mod residency; pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; diff --git a/drivers/gpu/drm/asahi/mmu/m3.rs b/drivers/gpu/drm/asahi/mmu/m3.rs new file mode 100644 index 00000000000000..a63101a8134268 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/m3.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The VM and binding helpers of the M3 T8122 runtime (`crate::m3`), which 7.1.12's `mmu.rs` +//! provided for it. + +use super::*; + +impl Vm { + /// Whether `other` is the same DRM VM. + pub(crate) fn same(&self, other: &Vm) -> bool { + self.id == other.id + } + + /// Copy the GPU-readable bytes this VM maps at `iova`, through the point of coherency + /// ([`Vm::read_user`]). M3 compute validates the client control stream before publishing it. + pub(crate) fn read_bytes(&self, iova: u64, out: &mut [u8]) -> Result { + if out.is_empty() { + return Err(EINVAL); + } + self.read_user(iova, out) + } + + /// Snapshot the first `out.len()` TTBAT entries as `(ttb0, ttb1)` pairs. The TTBAT is + /// ordinary memory, so this reads no GPU register and is safe with the cores gated. + pub(crate) fn context_roots(&self, out: &mut [(u64, u64)]) -> Result { + let inner = lock_vm(&self.gpuvm, None, false)?; + let shared = inner.uat_inner.lock(); + let ttbs = shared.ttbs(); + for (slot, entry) in out.iter_mut().enumerate().take(UAT_NUM_CTX) { + *entry = ( + ttbs[slot].ttb0.load(Ordering::Acquire), + ttbs[slot].ttb1.load(Ordering::Acquire), + ); + } + Ok(()) + } + + /// Map `size` bytes of MMIO or DMA memory at `phys` into this Vm, at `iova`, or at a free + /// address in `range` when `iova` is `None`. Unlike [`Vm::map_io`] this logs nothing on + /// success: the runtime maps job views on every submission. + fn map_io_m3_inner( + &self, + iova: Option, + range: Range, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + if size == 0 || (iova.unwrap_or(0) as usize | phys | size) & UAT_PGMSK != 0 { + pr_err!( + "MMU: M3 KernelMapping {:#x}:{:#x} -> {:#x?} is not page-aligned\n", + phys, + size, + iova + ); + return Err(EINVAL); + } + // SAFETY: As in `map_io`, the exec lock is only used for this mapping. + let mut vm_exec = unsafe { self.gpuvm.exec_lock(None, false)? }; + let inner = vm_exec.unique_gpuvm.data(); + let payload = KernelMappingInner { + owner: self.gpuvm.clone(), + uat_inner: inner.uat_inner.clone(), + prot, + bo: None, + _gem: None, + offset: 0, + mapped_size: size, + }; + let node = match iova { + Some(iova) => inner.mm.reserve_node(payload, iova, size as u64, 0)?, + None => inner.mm.insert_node_in_range( + payload, + size as u64, + UAT_PGSZ as u64, + 0, + range.start, + range.end, + mm::InsertMode::Best, + )?, + }; + let iova = node.start(); + let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; + inner.page_table.prepare_map(iova..end)?; + let mutation = inner.mapping_mutation(); + let ret = inner + .page_table + .map_pages(iova..end, phys as PhysicalAddr, prot, false); + if ret.is_err() { + inner.rollback_failed_map(iova, size as u64); + } + // Drop the exec lock before the node, as in `map_io`. + drop(mutation); + let _ = inner; + ret?; + Ok(KernelMapping(node)) + } + + /// [`Vm::map_io`] at `iova`, without the per-mapping log line. + pub(crate) fn map_io_m3( + &self, + iova: u64, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + self.map_io_m3_inner(Some(iova), 0..0, phys, size, prot) + } + + /// [`Vm::map_io`] at a free address in `range`. + pub(crate) fn map_io_in_range( + &self, + range: Range, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + self.map_io_m3_inner(None, range, phys, size, prot) + } +} + +impl VmBind { + /// The bound `Vm` itself, for diagnostics that need to walk its tables. + pub(crate) fn vm(&self) -> &Vm { + &self.0 + } + + /// Whether this retained binding belongs to the supplied DRM VM. + pub(crate) fn matches(&self, vm: &Vm) -> bool { + self.0.id == vm.id + } + + /// Physical base of the bound VM's page-table root. + pub(crate) fn root(&self) -> u64 { + self.0.ttb() + } + + /// The bound VM's driver-assigned id. + pub(crate) fn vm_id(&self) -> u64 { + self.0.id + } +} diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 5f7e572a3a4622..87ec3bd73658bb 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -318,6 +318,18 @@ impl RtKit { }) } + /// Returns whether the coprocessor is running (not crashed, IOP and AP power states on). + pub fn is_running(&self) -> bool { + // SAFETY: `rtk` is valid per the type invariant. The call only reads state. + unsafe { bindings::apple_rtkit_is_running(self.rtk) } + } + + /// Returns whether the coprocessor has reported a crash. + pub fn is_crashed(&self) -> bool { + // SAFETY: `rtk` is valid per the type invariant. The call only reads state. + unsafe { bindings::apple_rtkit_is_crashed(self.rtk) } + } + /// Checks if an endpoint is present pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } diff --git a/rust/kernel/sync/completion.rs b/rust/kernel/sync/completion.rs index 35ff049ff07830..095a51627edfe2 100644 --- a/rust/kernel/sync/completion.rs +++ b/rust/kernel/sync/completion.rs @@ -90,6 +90,28 @@ impl Completion { self.inner.get() } + /// Publish one completion token, including from interrupt context. + #[inline] + pub fn complete(&self) { + // SAFETY: The pinned completion remains initialized for this call. + unsafe { bindings::complete(self.as_raw()) }; + } + + /// Consume one pending token without blocking. + #[inline] + pub fn try_wait_for_completion(&self) -> bool { + // SAFETY: The pinned completion remains initialized for this call. + unsafe { bindings::try_wait_for_completion(self.as_raw()) } + } + + /// Wait for a token for at most `timeout` jiffies; return whether one arrived. + #[inline] + pub fn wait_for_completion_timeout(&self, timeout: crate::time::Jiffies) -> bool { + let timeout = timeout.min(core::ffi::c_long::MAX as _); + // SAFETY: The completion remains live while this method waits. + unsafe { bindings::wait_for_completion_timeout(self.as_raw(), timeout as _) != 0 } + } + /// Signal all tasks waiting on this completion. /// /// This method wakes up all tasks waiting on this completion; after this operation the From d8ecb2e706378e853c4a749fb8bb84a1d862d123 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 00:24:18 -0400 Subject: [PATCH 3/8] pmdomain: apple: pmp-report: keep the T8122 14.x GPU readiness for the M3 runtime The 7.2 rebase replaced the T8122 14.x branch of apple_pmp_report_wait_supplier_ready() with a T6030-only one, so on an M3 MacBook Air every GPU power vote returns -ENODEV even with the PMP running. With the new hidden APPLE_T8122_GPU_PMP_LINK, which DRM_ASAHI_M3_T8122 selects (as DRM_ASAHI_G15 selects APPLE_T6030_GPU_PMP_LINK), wait for the unique T8122 report's display request (DISP, reg 7) to be ready and acknowledged before AGX, as on the 7.1.12 kernel the runtime was qualified with. The PMP graph and supplier host tests copy the dispatch, so they copy the T8122 wait too, built off there; their mutations still target the T6030 path. Generated-by: LLM Co-Authored-By: Claude Opus 5.5 Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/Kconfig | 1 + drivers/pmdomain/apple/pmp-report.c | 43 ++++++++++++++++++++++++- drivers/soc/apple/Kconfig | 9 ++++++ tools/asahi/t6030-pmp-graph-tests.c | 1 + tools/asahi/t6030-pmp-graph-tests.py | 12 +++++-- tools/asahi/t6030-pmp-supplier-tests.py | 12 +++++++ 6 files changed, 74 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 4cfe26f28b8918..3f60caebdc11b2 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -41,6 +41,7 @@ config DRM_ASAHI_M3_T8122 depends on DRM_ASAHI # The runtime adopts the firmware's UAT through the G15 backend's handoff helpers. depends on DRM_ASAHI_G15 + select APPLE_T8122_GPU_PMP_LINK select CRYPTO_LIB_SHA256 default n help diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index b07bb9081d6c57..d39f6759c5d56c 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -232,6 +232,44 @@ static int apple_pmp_report_wait_t6030_14(struct device *supplier, unsigned int return ret; } +/* + * The T8122 (M3 MacBook Air) GPU on its 14.8.3 firmware, for the M3 runtime + * (APPLE_T8122_GPU_PMP_LINK): the unique T8122 report linked to this PMP, and + * its display request (DISP, reg 7) READY and acknowledged before AGX, as on + * the 7.1.12 kernel that runtime was qualified with. + */ +static int apple_pmp_report_wait_t8122_14(struct device *supplier, unsigned int timeout_ms) +{ + struct device_node *np, *report = NULL, *pmp, *entry; + int ret; + + if (!IS_ENABLED(CONFIG_APPLE_T8122_GPU_PMP_LINK) || + !of_machine_is_compatible("apple,t8122") || + !of_device_is_compatible(supplier->of_node, "apple,t8122-pmp-v2")) + return -ENODEV; + for_each_compatible_node(np, NULL, "apple,t8122-pmp-v2-report") { + if (report) { + of_node_put(np); + of_node_put(report); + return -ENODEV; + } + report = of_node_get(np); + } + if (!report || !of_device_is_available(report) || + of_property_count_u32_elems(report, "apple,pmp") != 1) { + of_node_put(report); + return -ENODEV; + } + pmp = of_parse_phandle(report, "apple,pmp", 0); + entry = apple_pmp_report_t6030_entry(report); + ret = pmp == supplier->of_node && entry ? + apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms)) : -ENODEV; + of_node_put(entry); + of_node_put(pmp); + of_node_put(report); + return ret; +} + /* The native 25G83 PMP starts through its supplier topology, after requests * are seeded. Its report is pinned against removal while readiness is read. */ @@ -244,8 +282,11 @@ int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int t int ret; /* A declared report always takes the existing native25 path below. */ - if (!np && !of_property_present(supplier->of_node, "apple,pmp-report")) + if (!np && !of_property_present(supplier->of_node, "apple,pmp-report")) { + if (of_machine_is_compatible("apple,t8122")) + return apple_pmp_report_wait_t8122_14(supplier, timeout_ms); return apple_pmp_report_wait_t6030_14(supplier, timeout_ms); + } if (!np || !of_device_is_compatible(np, "apple,j613-25g83-pmp-report")) { of_node_put(np); return -ENODEV; diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index af85f95e1a2014..61a46c92ee3f17 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -234,6 +234,15 @@ config APPLE_T6030_DISPLAY_GATE Say 'y' if you want the native display on M3 Pro MacBook Pros. +config APPLE_T8122_GPU_PMP_LINK + bool + depends on APPLE_PMP_REPORT + default n + help + Let T8122 (M3 MacBook Air) GPU power commands wait for the display + request the PMP report acknowledges on the 14.8.3 firmware, as the + M3 runtime of DRM_ASAHI_M3_T8122 requires. + config APPLE_T6030_GPU_PMP_LINK bool depends on APPLE_T6030_DISPLAY_GATE diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c index 37fdc54a057d98..e8a6bac35a9a50 100644 --- a/tools/asahi/t6030-pmp-graph-tests.c +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -19,6 +19,7 @@ typedef uint8_t u8; #define BIT(n) (1U << (n)) #define IS_ENABLED(x) (x) #define CONFIG_APPLE_T6030_GPU_PMP_LINK 1 +#define CONFIG_APPLE_T8122_GPU_PMP_LINK 0 #define PMP_REPORT_READY 1 #define GFP_KERNEL 0 #define GENPD_FLAG_ACTIVE_WAKEUP 1 diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py index dcef7dcc4c2504..50cea55ee727ac 100644 --- a/tools/asahi/t6030-pmp-graph-tests.py +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -47,11 +47,17 @@ def main(): names.insert(names.index('static struct device_node *apple_pmp_report_t6030_entry('), 'static int apple_pmp_report_wait_t6030_bound(') offsets = re.search(r'static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030 = \{.*?\n\};', source, re.S)[0] functions = '\n'.join(extract.function(source, n) for n in names) + # The T8122 14.x wait (CONFIG_APPLE_T8122_GPU_PMP_LINK=0 here) precedes the dispatch; it is + # added after the mutations are placed, so they target the T6030 path only. + t8122 = extract.function(source, 'static int apple_pmp_report_wait_t8122_14(') + '\n' + dispatch = 'int apple_pmp_report_wait_supplier_ready(' + def with_t8122(text): + return text.replace(dispatch, t8122 + dispatch, 1) validation = re.sub(r'^#include .*\n', '', (ROOT / 'drivers/pmdomain/apple/pmp-report-validation.h').read_text(), flags=re.M) code = offsets + '\n' + validation + '\n' + functions # The actual overlay must still describe a halted supplier and disabled DMA/mailbox. assert overlay.count('status = \"disabled\";') >= 3 - (args.output / 'production.h').write_text(code) + (args.output / 'production.h').write_text(with_t8122(code)) (args.output / 'overlay.h').write_text('static const struct overlay_entry overlay_entries[] = {\n' + entry_data + '\n};\n') compiler = shlex.split(os.environ.get('CC', 'clang')) command = compiler + ['-std=gnu11', '-g', '-O1', '-Wall', '-Wextra', '-Werror', @@ -83,13 +89,13 @@ def main(): ] for name, old, new in controls: assert code.count(old) == 1, (name, code.count(old)) - (args.output / 'production.h').write_text(code.replace(old, new)) + (args.output / 'production.h').write_text(with_t8122(code.replace(old, new))) built = subprocess.run(command, capture_output=True, text=True) # A mutant must compile; compilation failure is not regression evidence. ran = subprocess.run([str(args.output / 'graph')], capture_output=True, text=True) if not built.returncode else None (args.output / (name + '.log')).write_text(built.stdout + built.stderr + (ran.stdout + ran.stderr if ran else '')) mutations.append({'name': name, 'build_rc': built.returncode, 'run_rc': ran.returncode if ran else None}) - (args.output / 'production.h').write_text(code) + (args.output / 'production.h').write_text(with_t8122(code)) assert all(m['build_rc'] == 0 and m['run_rc'] != 0 for m in mutations), mutations (args.output / 'receipt.json').write_text(json.dumps({'source_ref': args.ref, 'source_sha256': hashlib.sha256(code.encode()).hexdigest(), 'overlay_sha256': hashlib.sha256(overlay.encode()).hexdigest(), 'overlay_entries': entries, diff --git a/tools/asahi/t6030-pmp-supplier-tests.py b/tools/asahi/t6030-pmp-supplier-tests.py index 3379b2911f4804..927bcb8c5e815f 100755 --- a/tools/asahi/t6030-pmp-supplier-tests.py +++ b/tools/asahi/t6030-pmp-supplier-tests.py @@ -54,6 +54,7 @@ def run(out, name, code, compiler, configs=(0, 1)): command = compiler + ["-std=gnu11", "-g", "-O1", "-Wall", "-Wextra", "-Werror", "-fsanitize=address,undefined", "-fno-omit-frame-pointer", f"-DCONFIG_APPLE_T6030_GPU_PMP_LINK={config}", + "-DCONFIG_APPLE_T8122_GPU_PMP_LINK=0", "-I", str(dest), str(ROOT / "tools/asahi/t6030-pmp-supplier-tests.c"), "-o", str(binary)] @@ -144,6 +145,17 @@ def main(): assert code.count(old) == 1, (name, code.count(old)) variants.append((name, code.replace(old, new))) variants.append(("control-after", code)) + # The T8122 14.x wait is built with CONFIG_APPLE_T8122_GPU_PMP_LINK=0 here; it is added to + # every variant after the T6030 mutations are placed, so they target the T6030 path only. + dispatch = "int apple_pmp_report_wait_supplier_ready(" + t8122 = function(report, "static int apple_pmp_report_wait_t8122_14(") + "\n" + def with_t8122(source): + if dispatch not in source: + return source + return source.replace(dispatch, t8122 + dispatch, 1) + variants = [(name, with_t8122(source)) for name, source in variants] + if baseline_code and "apple_pmp_report_wait_t8122_14(" in baseline_code: + baseline_code = with_t8122(baseline_code) results = [run(args.output, name, source, compiler) for name, source in variants] if baseline_code: results.append(run(args.output, "control-prior-config-off", baseline_code, compiler, (0,))) From 8e864a57c40e4b6d74a18afe1d4b6474b69031f8 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:52 -0400 Subject: [PATCH 4/8] drm/asahi: m3: map each VM's dummy page where the firmware expects it 7.1.12 mapped each VM's dummy page at the backend's unknown_page(), which the M3 runtime returned as 0x2ff_ffff_8000, the address its InitData gives the firmware (unk_30). On 7.2 file.rs maps it at uat_geometry().unk_page() instead, which is 0x3ff_ffff_8000 with the runtime's 42-bit geometry, so the firmware's unknown page was unmapped in every T8122 VM. Add a gpu::Gpu::unknown_page() hook, defaulting to the geometry's page so no other GPU changes, and have the M3 backend return RUNTIME_UNKNOWN_PAGE, which also ends its user range, so the two can't drift again. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/file.rs | 2 +- drivers/gpu/drm/asahi/gpu.rs | 4 ++++ drivers/gpu/drm/asahi/m3/initdata.rs | 2 +- drivers/gpu/drm/asahi/m3/m3_drm.rs | 9 ++++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 242f6cdcd3f838..f2f055931d29a5 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -549,7 +549,7 @@ impl File { } let dummy_mapping = dummy_obj.map_at( &vm, - gpu.uat_geometry().unk_page(), + gpu.unknown_page(), mmu::PROT_GPU_SHARED_RW, true, )?; diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index b9283efb470a93..a06b5da5ed6b14 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -344,6 +344,10 @@ pub(crate) trait Gpu: Send + Sync { fn admit_vm(&self, _dev: &kernel::device::Device, _params_size: u64, _flags: u32) -> Result { Ok(()) } + /// GPU VA of each VM's dummy page, which must match the address the firmware was given. + fn unknown_page(&self) -> u64 { + self.uat_geometry().unk_page() + } /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. diff --git a/drivers/gpu/drm/asahi/m3/initdata.rs b/drivers/gpu/drm/asahi/m3/initdata.rs index 2348d075a731cf..ab0e1643468f70 100644 --- a/drivers/gpu/drm/asahi/m3/initdata.rs +++ b/drivers/gpu/drm/asahi/m3/initdata.rs @@ -122,7 +122,7 @@ impl G15Options { /// Start of the USC (shader code) window in the runtime backend's GPU VA layout. const RUNTIME_USC_BASE: u64 = 0x10_0000_0000; /// The runtime backend's unknown-page address, the end of its 42-bit user VA range. -const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; +pub(crate) const RUNTIME_UNKNOWN_PAGE: u64 = 0x2ff_ffff_8000; /// Maximum number of entries of a firmware performance-state table, including the off state. const PSTATE_TABLE_ENTRIES: usize = 16; diff --git a/drivers/gpu/drm/asahi/m3/m3_drm.rs b/drivers/gpu/drm/asahi/m3/m3_drm.rs index 2d93823e67d015..dc8f1a074e809c 100644 --- a/drivers/gpu/drm/asahi/m3/m3_drm.rs +++ b/drivers/gpu/drm/asahi/m3/m3_drm.rs @@ -232,7 +232,14 @@ impl gpu::Gpu for Backend { value as u32 } fn uat_geometry(&self) -> mmu::UatGeometry { mmu::UatGeometry { ias: 42, oas: 42 } } - fn user_range(&self) -> Range { mmu::UAT_PGSZ as u64..0x2ff_ffff_8000 } + fn user_range(&self) -> Range { + mmu::UAT_PGSZ as u64..crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + } + /// The dummy page sits where InitData tells the firmware it is (unk_30), as on 7.1.12; the + /// 42-bit geometry's own unk_page() is a different address. + fn unknown_page(&self) -> u64 { + crate::m3::initdata::RUNTIME_UNKNOWN_PAGE + } fn tracked_submission(&self) -> bool { true } /// Only M3-aware userspace may create a VM (7.1.12's per-file client gate). fn admit_vm(&self, dev: &kernel::device::Device, params_size: u64, flags: u32) -> Result { From 75c5823f60b3f4c060e398783b9388eecd56e262 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:53 -0400 Subject: [PATCH 5/8] drm/asahi: m3: quarantine a failed job's VM before signalling its fence Packet::finish signalled an error fence while the packet could still hold its VM job guard, and nothing on the M3 path quarantined it, so after a timeout userspace could unmap and close memory that work the firmware had not provably stopped was using. Unmap invalidates the TLB, so this ended in a GPU fault rather than freed memory, as on 7.1.12, but 7.2's shared job lifetime can block it. As G15 does, quarantine a guard that is still held before the error fence becomes visible: mapping changes wait until the runtime drops the packet. Guards already released (work never published, or retired) are unaffected. Fix the comment that said this already happened. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_submit.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_submit.rs b/drivers/gpu/drm/asahi/m3/m3_submit.rs index 326e5148083820..ca16e18f10fbf5 100644 --- a/drivers/gpu/drm/asahi/m3/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3/m3_submit.rs @@ -84,12 +84,19 @@ impl Packet { // Claim error/signal ownership once; this flag is NOT proof that the // fence is signaled or that hardware DMA has stopped. if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } - if let Err(e)=result {if let Some(status) = self.vm.status() { status.record(e); }self.completion.set_error(e);} + if let Err(e)=result { + // A guard still held here belongs to work the firmware may still run: block + // mapping changes before the error fence is visible (as G15 does), so userspace + // can't unmap memory under it. It stays with the packet until the runtime drops + // the packet, after the firmware has provably stopped. + if let Some(guard)=self.vm_job.lock().as_ref() {guard.quarantine();} + if let Some(status) = self.vm.status() { status.record(e); } + self.completion.set_error(e); + } // A scheduler job/fence and the runtime's last packet can outlive // hardware retirement. They must not keep the VM's active-job count // nonzero: GEM closes otherwise accumulate across completed frames. - // On failure the runtime quarantines this guard with the packet until - // firmware has provably stopped; signalling an error is not retirement. + // Signalling an error is not retirement. crate::m3::agx_memory::publish(); let profile=crate::debug::debug_enabled(crate::debug::DebugFlags::M3PassTiming); let before=if profile {crate::m3::m3_runtime::physical_counter()} else {0}; From d2666d8ca7a528452da6b304dbcb709941956074 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:53 -0400 Subject: [PATCH 6/8] drm/asahi: m3: keep the adopted UAT until ASC stops on a failed start Once adopt_uat() points TTBAT context 0 at the kernel-owned lower root, a failure in Config::new() or the window sync dropped the UAT and its window inside the start closure, before stop_asc(), and the branch that retains everything when ASC fails to stop could not retain tables that were already freed. Keep both outside the closure: drop them only after ASC has stopped, and leak them with the rest when it does not. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_runtime.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_runtime.rs b/drivers/gpu/drm/asahi/m3/m3_runtime.rs index b12fc73f9f248d..28bbea3c86889b 100644 --- a/drivers/gpu/drm/asahi/m3/m3_runtime.rs +++ b/drivers/gpu/drm/asahi/m3/m3_runtime.rs @@ -997,6 +997,9 @@ impl Runtime { } // Whether the coprocessor runs and offers its endpoints, for the T8122 verdict. let mut started=false; + // The adopted UAT and its window live outside the closure: once TTBAT context 0 names + // the kernel-owned lower root, a failure must not free those tables before ASC stops. + let mut adopted=None; let prepared=(|| -> Result<_> { device.start_asc(pdev)?; Pin::new(&mut transport).wake()?; @@ -1010,23 +1013,28 @@ impl Runtime { // TTBAT of the admitted reservations, the firmware kernel root (`pagetables`) as // every context's upper half, and the host window (root entry 2) through the // shared L2. The RTKit is awake and no initdata or GPU job has been published. - let (uat,mut window)=adopt_uat(&drm,device.soc(),&device.firmware().resources)?; - let config=Config::new(&drm,&uat,device.firmware(),&contents)?; + let (uat,window)=adopted.insert(adopt_uat(&drm,device.soc(),&device.firmware().resources)?); + let config=Config::new(&drm,uat,device.firmware(),&contents)?; // Link every kernel table the configuration created before the firmware sees an // address in it, then drop any translation cached before they existed. - window.sync_all(&uat)?; + window.sync_all(uat)?; crate::mem::sync();crate::mem::tlbi_all();crate::mem::sync(); - Ok((uat,window,config)) + Ok(config) })(); - let (uat,window,config)=match prepared { - Ok(v)=>v, - Err(e)=>{ + let (uat,window,config)=match (prepared,adopted.take()) { + (Ok(config),Some((uat,window)))=>(uat,window,config), + (prepared,adopted)=>{ + let e=prepared.err().unwrap_or(EINVAL); crate::m3::t8122_start::prepare_verdict(pdev.as_ref(),device.soc(),started,e); state.health.mark_failed(); if device.stop_asc().is_err() { unsafe {kernel::bindings::__module_get(crate::THIS_MODULE.as_ptr())}; core::mem::forget(transport);core::mem::forget(state); + core::mem::forget(adopted); core::mem::forget(drm);core::mem::forget(device); + } else { + // ASC has stopped: the firmware no longer walks the adopted tables. + drop(adopted); } return Err(e); } From e8023bba60e6b1c8810d2b617fba1edb3940c5c8 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:54 -0400 Subject: [PATCH 7/8] drm/asahi: m3: don't let the compute stream debug read fail a job Compute::new() read 48 bytes of the client's CDM stream for a debug message, whatever the stream's length, and failed the job if the read did. A short stream ending at a mapped page boundary with the next page unmapped passed the stream checks but then failed its first submission, even with debug logging off (as on 7.1.12). Read no further than the stream, and only log when the read succeeds. Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- drivers/gpu/drm/asahi/m3/m3_compute.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/m3/m3_compute.rs b/drivers/gpu/drm/asahi/m3/m3_compute.rs index eab8bf71f7bcdf..34aa0c0d2cd3fb 100644 --- a/drivers/gpu/drm/asahi/m3/m3_compute.rs +++ b/drivers/gpu/drm/asahi/m3/m3_compute.rs @@ -96,9 +96,12 @@ impl Compute { objects[storage::COMMAND].gpu_va()?,control.base,control.end,objects[storage::POOL].va()); // The M3 compute stream uses absolute IOTO addresses and does not // include the USC_EXEC_BASE_CP register. + // Diagnostic only: read no further than the stream, and never fail the job over it. let mut stream=[0u8;48]; - vm.read_bytes(control.base,&mut stream)?; - crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",stream); + let len=core::cmp::min(control.end.saturating_sub(control.base),stream.len() as u64) as usize; + if vm.read_bytes(control.base,&mut stream[..len]).is_ok() { + crate::cls_dev_dbg!(Compute,dev,"M3 client CDM bytes={:02x?}\n",&stream[..len]); + } agx_memory::publish(); // Finish the write-combining page-table stores before broadcasting // the invalidation. From 4888ad0a0fa63995653268fa76c1fdf5ceba7aa0 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 23 Oct 2026 11:00:54 -0400 Subject: [PATCH 8/8] tools/asahi: test the T8122 14.x PMP wait in the supplier suite The supplier suite built apple_pmp_report_wait_t8122_14() only with APPLE_T8122_GPU_PMP_LINK off, so every case returned at its first check. Build it also as the M3 runtime does (T6030 and T8122 links on) and run the T8122 graph: the unique, available report linked to this PMP, with DISP (reg 7) acked, and its failures (machine, supplier and report compatibles, a duplicate or disabled report, the apple,pmp shape and identity, a missing, duplicate, disabled, incompatible or no-ack DISP entry, the managed start and its readiness). With the link off, a well-formed T8122 graph is refused. Mutants cover the wait's config scope, compatibles, report checks, readiness result and entry reference, and the dispatch's machine scope in both directions. Reported-by: Marcelo Alcantara Generated-by: LLM Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Scott Jones --- tools/asahi/t6030-pmp-supplier-tests.c | 60 +++++++++++++++++++++++-- tools/asahi/t6030-pmp-supplier-tests.py | 47 +++++++++++++++---- 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c index b581f3863a2323..bcf94e6244e690 100644 --- a/tools/asahi/t6030-pmp-supplier-tests.c +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -460,6 +460,17 @@ static void reset(void) lockable = present_device = link_ok = 1; status_word = PMP_REPORT_READY; } +/* The T8122 (M3 Air) 14.x graph: the same report shape, with T8122 compatibles. */ +static void t8122(void) +{ + machine = "apple,t8122"; + gate_soc = &gate_t8122; + supplier_node.compatible = "apple,t8122-pmp-v2"; + other_node.compatible = "apple,t8122-pmp-v2"; + gpu.compatible = "apple,agx-t8122"; + report14.compatible = "apple,t8122-pmp-v2-report"; + apple_pmp_starts.first = &managed_start; +} static int power(void) { return apple_pmp_set_device_power(0x0f, 5, 1); @@ -489,6 +500,7 @@ int main(void) (void)gate_find_one; (void)apple_pmp_report_t6030_entry; (void)apple_pmp_report_wait_t6030_14; + (void)apple_pmp_report_wait_t8122_14; reset(); assert(apple_pmp_offsets_t6030.starts_pmp == !!CONFIG_APPLE_T6030_GPU_PMP_LINK); @@ -542,9 +554,50 @@ int main(void) RUN((void)0, -ENODEV, 0, 0); assert(!find_calls); #endif - /* T8122 current14 remains unsupported by this supplier API. */ + /* A T8122 supplier never takes the T6030 report. */ RUN(machine = "apple,t8122"; supplier_node.compatible = "apple,t8122-pmp-v2", -ENODEV, 0, 0); +#if CONFIG_APPLE_T8122_GPU_PMP_LINK + /* The T8122 14.x wait: one available report linked to this PMP, DISP (reg 7) acked. */ + RUN(t8122(), 0, 0, 1); + RUN(t8122(); report14.child = &entry7; entry7.sibling = &entry17; + entry17.sibling = NULL, 0, 0, 1); + RUN(t8122(); apple_pmp_starts.first = NULL, -ENODEV, 0, 0); + RUN(t8122(); apple_pmp_starts.first = NULL; + apple_pmp_reports.first = NULL, -EPROBE_DEFER, 0, 0); + RUN(t8122(); managed_start.stopping = true, -ENODEV, 0, 0); + RUN(t8122(); ready_result = -ETIMEDOUT, -ETIMEDOUT, 0, 0); + RUN(t8122(); passive_words[apple_pmp_offsets_t6030.status / 8] = 0, -EIO, + 0, 0); + RUN(t8122(); passive_words[apple_pmp_offsets_t6030.actual / 8] = 0, -EIO, + 0, 0); + RUN(t8122(); power_result = -EIO, -EIO, 0, 1); + RUN(t8122(); machine = "apple,t6030", -ENODEV, 0, 0); + RUN(t8122(); machine = "apple,t8112", -ENODEV, 0, 0); + RUN(t8122(); supplier_node.compatible = "apple,t6030-pmp-v2", -ENODEV, 0, + 0); + RUN(t8122(); report14.compatible = NULL, -ENODEV, 0, 0); + RUN(t8122(); report14.compatible = "apple,t6030-pmp-v2-report", -ENODEV, + 0, 0); + RUN(t8122(); extra_report = report14, -ENODEV, 0, 0); + RUN(t8122(); extra_report = report14; extra_report.available = false, + -ENODEV, 0, 0); + RUN(t8122(); report14.available = false, -ENODEV, 0, 0); + RUN(t8122(); report14.has_pmp = false, -ENODEV, 0, 0); + RUN(t8122(); report14.pmp_cells = 0, -ENODEV, 0, 0); + RUN(t8122(); report14.pmp_cells = 2, -ENODEV, 0, 0); + RUN(t8122(); report14.pmp = NULL, -ENODEV, 0, 0); + RUN(t8122(); report14.pmp = &other_node, -ENODEV, 0, 0); + RUN(t8122(); entry17.sibling = NULL, -ENODEV, 0, 0); + RUN(t8122(); entry7.reg = 8, -ENODEV, 0, 0); + RUN(t8122(); entry7.available = false, -ENODEV, 0, 0); + RUN(t8122(); entry7.compatible = "apple,other", -ENODEV, 0, 0); + RUN(t8122(); entry7.no_ack = true, -ENODEV, 0, 0); + RUN(t8122(); entry17.reg = 7, -ENODEV, 0, 0); +#else + /* Without APPLE_T8122_GPU_PMP_LINK, even a well-formed T8122 graph is refused. */ + RUN(t8122(), -ENODEV, 0, 0); +#endif /* A declared report, including malformed/unresolved data, never falls back. */ RUN(supplier_node.has_report = true; supplier_node.report_cells = 0, -ENODEV, 0, 0); @@ -684,7 +737,8 @@ int main(void) assert(apple_pmp_link_device(&consumer) == -EINVAL); balanced(); case_count++; - printf("PASS: %d T6030 PMP cases, config=%d; reference, lock and command boundaries checked\n", - case_count, CONFIG_APPLE_T6030_GPU_PMP_LINK); + printf("PASS: %d T6030/T8122 PMP cases, config=%d/%d; reference, lock and command boundaries checked\n", + case_count, CONFIG_APPLE_T6030_GPU_PMP_LINK, + CONFIG_APPLE_T8122_GPU_PMP_LINK); return 0; } diff --git a/tools/asahi/t6030-pmp-supplier-tests.py b/tools/asahi/t6030-pmp-supplier-tests.py index 927bcb8c5e815f..d7934acd3ae98c 100755 --- a/tools/asahi/t6030-pmp-supplier-tests.py +++ b/tools/asahi/t6030-pmp-supplier-tests.py @@ -44,17 +44,23 @@ def no_core(): resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) -def run(out, name, code, compiler, configs=(0, 1)): +# (CONFIG_APPLE_T6030_GPU_PMP_LINK, CONFIG_APPLE_T8122_GPU_PMP_LINK): off, G15 only, and G15 +# with the M3 runtime (DRM_ASAHI_M3_T8122 depends on DRM_ASAHI_G15). +CONFIGS = ((0, 0), (1, 0), (1, 1)) + + +def run(out, name, code, compiler, configs=CONFIGS): dest = out / name dest.mkdir() (dest / "t6030-pmp-production.h").write_text(code) results = [] - for config in configs: + for t6030, t8122 in configs: + config = f"{t6030}{t8122}" binary = dest / f"config-{config}" command = compiler + ["-std=gnu11", "-g", "-O1", "-Wall", "-Wextra", "-Werror", "-fsanitize=address,undefined", "-fno-omit-frame-pointer", - f"-DCONFIG_APPLE_T6030_GPU_PMP_LINK={config}", - "-DCONFIG_APPLE_T8122_GPU_PMP_LINK=0", + f"-DCONFIG_APPLE_T6030_GPU_PMP_LINK={t6030}", + f"-DCONFIG_APPLE_T8122_GPU_PMP_LINK={t8122}", "-I", str(dest), str(ROOT / "tools/asahi/t6030-pmp-supplier-tests.c"), "-o", str(binary)] @@ -145,20 +151,43 @@ def main(): assert code.count(old) == 1, (name, code.count(old)) variants.append((name, code.replace(old, new))) variants.append(("control-after", code)) - # The T8122 14.x wait is built with CONFIG_APPLE_T8122_GPU_PMP_LINK=0 here; it is added to - # every variant after the T6030 mutations are placed, so they target the T6030 path only. + # The T8122 14.x wait mirrors the T6030 one, so it is added to every variant after the + # T6030 mutations are placed; its own mutants change only its text. dispatch = "int apple_pmp_report_wait_supplier_ready(" t8122 = function(report, "static int apple_pmp_report_wait_t8122_14(") + "\n" - def with_t8122(source): + def with_t8122(source, text=t8122): if dispatch not in source: return source - return source.replace(dispatch, t8122 + dispatch, 1) + return source.replace(dispatch, text + dispatch, 1) variants = [(name, with_t8122(source)) for name, source in variants] + if args.mutations: + t8122_mutations = [ + ("t8122-config-scope", "if (!IS_ENABLED(CONFIG_APPLE_T8122_GPU_PMP_LINK) ||", "if (false ||"), + ("t8122-supplier-compat", '!of_device_is_compatible(supplier->of_node, "apple,t8122-pmp-v2"))', "false)"), + ("t8122-report-compat", 'for_each_compatible_node(np, NULL, "apple,t8122-pmp-v2-report")', + 'for_each_compatible_node(np, NULL, "apple,t6030-pmp-v2-report")'), + ("t8122-duplicate-report", "\t\tif (report) {", "\t\tif (false) {"), + ("t8122-disabled-report", "!of_device_is_available(report) ||", "false ||"), + ("t8122-report-phandle-shape", 'of_property_count_u32_elems(report, "apple,pmp") != 1', "false"), + ("t8122-supplier-identity", "pmp == supplier->of_node && entry ?", "entry ?"), + ("t8122-readiness-error", "apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms)) : -ENODEV;", + "(apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms)), 0) : -ENODEV;"), + ("t8122-entry-reference", "\tof_node_put(entry);\n\tof_node_put(pmp);", "\tof_node_put(pmp);"), + ] + for name, old, new in t8122_mutations: + assert t8122.count(old) == 1, (name, t8122.count(old)) + variants.insert(-1, (name, with_t8122(code, t8122.replace(old, new)))) + # The dispatch owns the machine scope: T8122 takes the T8122 wait, every other SoC the + # T6030 one. (The wait's own machine check repeats it and cannot be reached otherwise.) + machine = 'if (of_machine_is_compatible("apple,t8122"))' + assert code.count(machine) == 1 + for name, new in (("t8122-dispatch-never", "if (false)"), ("t8122-dispatch-always", "if (true)")): + variants.insert(-1, (name, with_t8122(code.replace(machine, new)))) if baseline_code and "apple_pmp_report_wait_t8122_14(" in baseline_code: baseline_code = with_t8122(baseline_code) results = [run(args.output, name, source, compiler) for name, source in variants] if baseline_code: - results.append(run(args.output, "control-prior-config-off", baseline_code, compiler, (0,))) + results.append(run(args.output, "control-prior-config-off", baseline_code, compiler, ((0, 0),))) valid = True for item in results: control = item["name"].startswith("control-")