diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c index cdc812269542..2c2fc14dd10e 100644 --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -131,6 +131,7 @@ static struct trapframe proc0_tf; int early_boot = 1; int cold = 1; static int boot_el; +uintptr_t socdev_va; static uint64_t hcr_el2; struct kva_md_info kmi; diff --git a/sys/arm64/conf/GATEWAY b/sys/arm64/conf/GATEWAY new file mode 100644 index 000000000000..849607391a33 --- /dev/null +++ b/sys/arm64/conf/GATEWAY @@ -0,0 +1,99 @@ +# +# MONO-GATEWAY -- NXP LS1046A with DPAA1 + CAAM for OPNsense +# +# Based on GENERIC (like SMP-ARM) with hardware-specific additions: +# - DPAA1 packet acceleration (FMan/QMan/BMan) +# - CAAM crypto accelerator (JR + QI paths) +# - GPY115 PHY, SFP+, I2C sensors +# - Early printf via QorIQ ns16550 +# + +include GENERIC + +ident MONO-GATEWAY + +# Disable debug for production +nooptions INVARIANTS +nooptions INVARIANT_SUPPORT +nooptions WITNESS +nooptions WITNESS_SKIPSPIN +nooptions LOCK_PROFILING + +# OPNsense standard features (from SMP-ARM) +options DDB +options GEOM_BDE +options GEOM_ELI +options GEOM_MIRROR +options IPFIREWALL_DEFAULT_TO_ACCEPT +options IPFIREWALL_VERBOSE +options IPSTEALTH +options MROUTING +options PPS_SYNC +options TCP_SIGNATURE + +# Platform +options SOC_NXP_LS + +# ZFS (compiled in -- no FreeBSD loader with booti) +options ZFS + +# Root filesystem (ZFS pool "pool0") +options ROOTDEVNAME=\"zfs:pool0/ROOT/default\" + +# U-Boot (build booti compatible kernel) +options LINUX_BOOT_ABI + +# Disable diskid labels so GPT labels (/dev/gpt/*) work with ZFS +# (diskid provider prevents GPT from attaching to the raw device) +envvar kern.geom.label.disk_ident.enable="0" + +# Early printf for debugging +options SOCDEV_PA=0x2000000 +options EARLY_PRINTF=qoriq + +# eMMC (QorIQ ESDHC) +device sdhci + +# I2C (board-specific) +device vf_i2c # Freescale Vybrid / QorIQ I2C controller +device pca954x # NXP PCA9545/9546/9548 I2C mux/switch +device ina2xx # TI INA219/INA226/INA234 power monitor +# pcf2131 is built as an out-of-tree module in opnsense-deps/drivers/pcf2131/ + +# GPIO power-off (STUSB4500 USB-C PD reset) +device gpiopower + +# DPAA1 (FMan/QMan/BMan) -- NXP LS1046A packet acceleration +include "dpaa/config.dpaa_arm64" +device dpaa +options QMAN_CEETM_SUPPORT +device sff # Small Form Factor Transceivers (SFP+) + +# caam is built as an out-of-tree module in opnsense-deps/drivers/caam/ + +# Packet filter +device pf +device pflog +device pfsync +device carp + +# Networking +device lagg # Link aggregation +device if_bridge # Bridge interface +device enc # IPsec enc(4) interface +device gif # IPv6/IPv4 tunneling +device gre # GRE tunneling +device tuntap # Tunnel/TAP + +# OPNsense extras +device wlan_acl +device wlan_xauth +device u3g +device uftdi +device umodem +device uplcom +device uvisor +device uvscom + +# DTBs +makeoptions MODULES_EXTRA+="dtb/freescale" diff --git a/sys/arm64/conf/dpaa/config.dpaa_arm64 b/sys/arm64/conf/dpaa/config.dpaa_arm64 new file mode 100644 index 000000000000..47eb00bf0a88 --- /dev/null +++ b/sys/arm64/conf/dpaa/config.dpaa_arm64 @@ -0,0 +1,25 @@ + +files "dpaa/files.dpaa_arm64" + +makeoptions DPAA_COMPILE_CMD="${LINUXKPI_C} ${DPAAWARNFLAGS} \ + -Wno-cast-qual -Wno-unused-function -Wno-init-self -fms-extensions \ + -include $S/contrib/ncsw/build/dflags_arm64.h \ + -Wno-error=missing-prototypes \ + -I$S/contrib/ncsw/build/ \ + -I$S/contrib/ncsw/inc \ + -I$S/contrib/ncsw/inc/cores \ + -I$S/contrib/ncsw/inc/etc \ + -I$S/contrib/ncsw/inc/Peripherals \ + -I$S/contrib/ncsw/etc \ + -I$S/contrib/ncsw/Peripherals/BM \ + -I$S/contrib/ncsw/Peripherals/FM \ + -I$S/contrib/ncsw/Peripherals/FM/HC \ + -I$S/contrib/ncsw/Peripherals/FM/inc \ + -I$S/contrib/ncsw/Peripherals/FM/MAC \ + -I$S/contrib/ncsw/Peripherals/FM/Pcd \ + -I$S/contrib/ncsw/Peripherals/FM/Port \ + -I$S/contrib/ncsw/Peripherals/FM/Rtc \ + -I$S/contrib/ncsw/Peripherals/QM \ + -I$S/contrib/ncsw/inc/flib \ + -I$S/contrib/ncsw/inc/integrations \ + -I$S/contrib/ncsw/inc/ioctls" diff --git a/sys/arm64/conf/dpaa/files.dpaa_arm64 b/sys/arm64/conf/dpaa/files.dpaa_arm64 new file mode 100644 index 000000000000..9cfbb0c47d46 --- /dev/null +++ b/sys/arm64/conf/dpaa/files.dpaa_arm64 @@ -0,0 +1,131 @@ + +# NetCommSW drivers — ARM64 variant +# Identical source list to powerpc/conf/dpaa/files.dpaa except: +# - core.c -> core_arm64.c (ARM64 CORE_GetId etc.) +# - xx.c -> xx_arm64.c (no PowerPC ccsrbar/LAW/TLB deps) +# - if_dtsec* excluded (LS1046A uses mEMAC, not dTSEC) + +contrib/ncsw/etc/error.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/etc/ncsw_list.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/etc/ncsw_memcpy.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/etc/mm.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/etc/ncsw_mem.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/etc/sprint.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/BM/bm.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/BM/bman_low.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/BM/bm_pool.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/BM/bm_portal.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/SP/fm_sp.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD} -Wno-enum-conversion" +contrib/ncsw/Peripherals/FM/SP/fman_sp.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Rtc/fm_rtc.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Rtc/fman_rtc.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Port/fman_port.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Port/fm_port.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Port/fm_port_im.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fm_cc.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fm_ehash.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fm_kg.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fman_kg.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fm_manip.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fm_pcd.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fm_plcr.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fm_prs.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fman_prs.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/Pcd/fm_replic.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +# mEMAC — the MAC type used by LS1046A FManv3 (DPAA_VERSION >= 11) +contrib/ncsw/Peripherals/FM/MAC/memac.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/MAC/memac_mii_acc.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/MAC/fman_memac.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/MAC/fman_memac_mii_acc.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/MAC/fm_mac.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/HC/hc.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/fm_muram.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/FM/fm_ncsw.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD} -Wno-unused-but-set-variable" +contrib/ncsw/Peripherals/FM/fman_ncsw.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD} ${NO_WMISLEADING_INDENTATION}" +contrib/ncsw/Peripherals/QM/qm.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/Peripherals/QM/qm_portal_fqr.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/user/env/stdlib.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +# ARM64 replacements for xx.c and core.c (no PowerPC deps) +contrib/ncsw/user/env/xx_arm64.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +contrib/ncsw/user/env/core_arm64.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" + +# FreeBSD Wrappers +dev/dpaa/portals_common.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/bman_portals.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/bman.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/bman_fdt.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/qman_portals.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/qman.c optional dpaa \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/qman_fdt.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/fman.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/fman_chardev.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/fman_fdt.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +# mEMAC external MDIO controller (fsl,fman-memac-mdio) +dev/dpaa/memac_mdio.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +# Ethernet driver (reuses dTSEC wrapper with mEMAC via NCSW FM_MAC API) +dev/dpaa/if_dtsec.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/if_dtsec_fdt.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/if_dtsec_im.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +dev/dpaa/if_dtsec_rm.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +# Offline / Header Manipulation ports +dev/dpaa/dpaa_oh.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" +# CEETM egress QoS (Class-based Enqueue/Dequeue Traffic Management) +dev/dpaa/qman_ceetm.c optional dpaa fdt \ + no-depend compile-with "${DPAA_COMPILE_CMD}" diff --git a/sys/arm64/include/pio.h b/sys/arm64/include/pio.h new file mode 100644 index 000000000000..b33dc78bfdf3 --- /dev/null +++ b/sys/arm64/include/pio.h @@ -0,0 +1,237 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Mono Technologies Inc. + * + * I/O macros for ARM64 — equivalent to powerpc/include/pio.h. + * Provides in8/out8/../in64/out64 used by the NetCommSW (ncsw) DPAA1 driver. + * All accesses are memory-mapped volatile loads/stores with a DSB barrier + * to match the ordering semantics of the PowerPC "sync" in the original. + */ + +#ifndef _MACHINE_PIO_H_ +#define _MACHINE_PIO_H_ + +#include + +/* + * Full system data synchronisation barrier. + * Ensures completion of all preceding memory accesses before any + * subsequent access can begin. Equivalent to PowerPC "sync". + */ +#define arm64_iomb() __asm __volatile("dsb sy" : : : "memory") + +static __inline void +__outb(volatile u_int8_t *a, u_int8_t v) +{ + *a = v; + arm64_iomb(); +} + +static __inline void +__outw(volatile u_int16_t *a, u_int16_t v) +{ + *a = v; + arm64_iomb(); +} + +static __inline void +__outl(volatile u_int32_t *a, u_int32_t v) +{ + *a = v; + arm64_iomb(); +} + +static __inline void +__outll(volatile u_int64_t *a, u_int64_t v) +{ + *a = v; + arm64_iomb(); +} + +static __inline u_int8_t +__inb(volatile u_int8_t *a) +{ + u_int8_t v; + + v = *a; + arm64_iomb(); + return (v); +} + +static __inline u_int16_t +__inw(volatile u_int16_t *a) +{ + u_int16_t v; + + v = *a; + arm64_iomb(); + return (v); +} + +static __inline u_int32_t +__inl(volatile u_int32_t *a) +{ + u_int32_t v; + + v = *a; + arm64_iomb(); + return (v); +} + +static __inline u_int64_t +__inll(volatile u_int64_t *a) +{ + u_int64_t v; + + v = *a; + arm64_iomb(); + return (v); +} + +#define outb(a,v) (__outb((volatile u_int8_t *)(a), v)) +#define out8(a,v) outb(a,v) +#define outw(a,v) (__outw((volatile u_int16_t *)(a), v)) +#define out16(a,v) outw(a,v) +#define outl(a,v) (__outl((volatile u_int32_t *)(a), v)) +#define out32(a,v) outl(a,v) +#define outll(a,v) (__outll((volatile u_int64_t *)(a), v)) +#define out64(a,v) outll(a,v) +#define inb(a) (__inb((volatile u_int8_t *)(a))) +#define in8(a) inb(a) +#define inw(a) (__inw((volatile u_int16_t *)(a))) +#define in16(a) inw(a) +#define inl(a) (__inl((volatile u_int32_t *)(a))) +#define in32(a) inl(a) +#define inll(a) (__inll((volatile u_int64_t *)(a))) +#define in64(a) inll(a) + +/* Reverse-byte variants — ARM64 is LE, so these byte-swap for BE regs */ +static __inline void +__outwrb(volatile u_int16_t *a, u_int16_t v) +{ + *a = __builtin_bswap16(v); + arm64_iomb(); +} + +static __inline void +__outlrb(volatile u_int32_t *a, u_int32_t v) +{ + *a = __builtin_bswap32(v); + arm64_iomb(); +} + +static __inline u_int16_t +__inwrb(volatile u_int16_t *a) +{ + u_int16_t v; + + v = *a; + arm64_iomb(); + return (__builtin_bswap16(v)); +} + +static __inline u_int32_t +__inlrb(volatile u_int32_t *a) +{ + u_int32_t v; + + v = *a; + arm64_iomb(); + return (__builtin_bswap32(v)); +} + +#define out8rb(a,v) outb(a,v) +#define outwrb(a,v) (__outwrb((volatile u_int16_t *)(a), v)) +#define out16rb(a,v) outwrb(a,v) +#define outlrb(a,v) (__outlrb((volatile u_int32_t *)(a), v)) +#define out32rb(a,v) outlrb(a,v) +#define in8rb(a) inb(a) +#define inwrb(a) (__inwrb((volatile u_int16_t *)(a))) +#define in16rb(a) inwrb(a) +#define inlrb(a) (__inlrb((volatile u_int32_t *)(a))) +#define in32rb(a) inlrb(a) + +/* Bulk accessors */ +static __inline void +__outsb(volatile u_int8_t *a, const u_int8_t *s, size_t c) +{ + while (c--) + *a = *s++; + arm64_iomb(); +} + +static __inline void +__outsw(volatile u_int16_t *a, const u_int16_t *s, size_t c) +{ + while (c--) + *a = *s++; + arm64_iomb(); +} + +static __inline void +__outsl(volatile u_int32_t *a, const u_int32_t *s, size_t c) +{ + while (c--) + *a = *s++; + arm64_iomb(); +} + +static __inline void +__outsll(volatile u_int64_t *a, const u_int64_t *s, size_t c) +{ + while (c--) + *a = *s++; + arm64_iomb(); +} + +static __inline void +__insb(volatile u_int8_t *a, u_int8_t *d, size_t c) +{ + while (c--) + *d++ = *a; + arm64_iomb(); +} + +static __inline void +__insw(volatile u_int16_t *a, u_int16_t *d, size_t c) +{ + while (c--) + *d++ = *a; + arm64_iomb(); +} + +static __inline void +__insl(volatile u_int32_t *a, u_int32_t *d, size_t c) +{ + while (c--) + *d++ = *a; + arm64_iomb(); +} + +static __inline void +__insll(volatile u_int64_t *a, u_int64_t *d, size_t c) +{ + while (c--) + *d++ = *a; + arm64_iomb(); +} + +#define outsb(a,s,c) (__outsb((volatile u_int8_t *)(a), s, c)) +#define outs8(a,s,c) outsb(a,s,c) +#define outsw(a,s,c) (__outsw((volatile u_int16_t *)(a), s, c)) +#define outs16(a,s,c) outsw(a,s,c) +#define outsl(a,s,c) (__outsl((volatile u_int32_t *)(a), s, c)) +#define outs32(a,s,c) outsl(a,s,c) +#define outsll(a,s,c) (__outsll((volatile u_int64_t *)(a), s, c)) +#define outs64(a,s,c) outsll(a,s,c) +#define insb(a,d,c) (__insb((volatile u_int8_t *)(a), d, c)) +#define ins8(a,d,c) insb(a,d,c) +#define insw(a,d,c) (__insw((volatile u_int16_t *)(a), d, c)) +#define ins16(a,d,c) insw(a,d,c) +#define insl(a,d,c) (__insl((volatile u_int32_t *)(a), d, c)) +#define ins32(a,d,c) insl(a,d,c) +#define insll(a,d,c) (__insll((volatile u_int64_t *)(a), d, c)) +#define ins64(a,d,c) insll(a,d,c) + +#endif /* _MACHINE_PIO_H_ */ diff --git a/sys/arm64/qoriq/qoriq_dw_pci.c b/sys/arm64/qoriq/qoriq_dw_pci.c index 2323b4bc5e1f..20ca672d79fc 100644 --- a/sys/arm64/qoriq/qoriq_dw_pci.c +++ b/sys/arm64/qoriq/qoriq_dw_pci.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -71,7 +72,7 @@ struct qorif_dw_pci_softc { struct resource *irq_res; void *intr_cookie; struct qoriq_dw_pci_cfg *soc_cfg; - + bool big_endian; /* CCSR registers are BE */ }; static struct qoriq_dw_pci_cfg ls1043_cfg = { @@ -100,7 +101,7 @@ static struct ofw_compat_data compat_data[] = { {"fsl,ls1012a-pcie", (uintptr_t)&ls1012_cfg}, {"fsl,ls1028a-pcie", (uintptr_t)&ls2028_cfg}, {"fsl,ls1043a-pcie", (uintptr_t)&ls1043_cfg}, - {"fsl,ls1046a-pcie", (uintptr_t)&ls1012_cfg}, + {"fsl,ls1046a-pcie", (uintptr_t)&ls2028_cfg}, {"fsl,ls2080a-pcie", (uintptr_t)&ls2080_cfg}, {"fsl,ls2085a-pcie", (uintptr_t)&ls2080_cfg}, {"fsl,ls2088a-pcie", (uintptr_t)&ls2028_cfg}, @@ -141,10 +142,11 @@ static int qorif_dw_pci_get_link(device_t dev, bool *status) { struct qorif_dw_pci_softc *sc; - uint32_t reg; + uint32_t reg, raw; sc = device_get_softc(dev); - reg = pci_dw_dbi_rd4(sc->dev, sc->soc_cfg->pex_pf0_dgb); + raw = pci_dw_dbi_rd4(sc->dev, sc->soc_cfg->pex_pf0_dgb); + reg = sc->big_endian ? bswap32(raw) : raw; reg >>= sc->soc_cfg->ltssm_bit; reg &= 0x3F; *status = (reg == 0x11) ? true : false; @@ -198,6 +200,7 @@ qorif_dw_pci_attach(device_t dev) sc->node = node; sc->soc_cfg = (struct qoriq_dw_pci_cfg *) ofw_bus_search_compatible(dev, compat_data)->ocd_data; + sc->big_endian = OF_hasprop(node, "big-endian"); rid = 0; sc->dw_sc.dbi_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, @@ -234,6 +237,20 @@ qorif_dw_pci_attach(device_t dev) qorif_dw_pci_init(sc); + /* Report link status */ + { + uint32_t raw, ltssm; + bool link_up; + + raw = pci_dw_dbi_rd4(dev, sc->soc_cfg->pex_pf0_dgb); + ltssm = sc->big_endian ? bswap32(raw) : raw; + ltssm = (ltssm >> sc->soc_cfg->ltssm_bit) & 0x3F; + link_up = (ltssm == 0x11); + device_printf(dev, + "PCIe link: LTSSM=0x%02x (%s), PEX_PF0_DBG=0x%08x (raw), big_endian=%d\n", + ltssm, link_up ? "L0/up" : "not up", raw, sc->big_endian); + } + /* Setup interrupt */ if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_MISC | INTR_MPSAFE, qorif_dw_pci_intr, NULL, sc, &sc->intr_cookie)) { diff --git a/sys/conf/files b/sys/conf/files index 00816d17c700..4483fcf521e3 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1838,6 +1838,7 @@ dev/iicbus/rtc/rv3032.c optional rv3032 iicbus fdt dev/iicbus/rtc/rx8803.c optional rx8803 iicbus fdt dev/iicbus/rtc/s35390a.c optional s35390a dev/iicbus/sensor/htu21.c optional htu21 +dev/iicbus/sensor/ina2xx.c optional ina2xx dev/iicbus/sensor/lm75.c optional lm75 dev/iicbus/sensor/max44009.c optional max44009 dev/iicbus/gpio/pcf8574.c optional pcf8574 @@ -2401,6 +2402,7 @@ dev/mii/dp83822phy.c optional miibus | dp83822phy dev/mii/dp83867phy.c optional miibus | dp83867phy dev/mii/e1000phy.c optional miibus | e1000phy dev/mii/gentbi.c optional miibus | gentbi +dev/mii/gpyphy.c optional miibus | gpyphy dev/mii/icsphy.c optional miibus | icsphy dev/mii/ip1000phy.c optional miibus | ip1000phy dev/mii/jmphy.c optional miibus | jmphy diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index cf2e1d22da88..b22a047f8f5b 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -636,8 +636,8 @@ dev/iicbus/controller/vybrid/vf_i2c_acpi.c optional vf_i2c iicbus acpi soc_nxp_l dev/iicbus/controller/vybrid/vf_i2c_fdt.c optional vf_i2c iicbus fdt soc_nxp_ls arm64/qoriq/qoriq_dw_pci.c optional pci fdt soc_nxp_ls arm64/qoriq/qoriq_gpio_pic.c optional gpio fdt soc_nxp_ls -arm64/qoriq/qoriq_therm.c optional pci fdt soc_nxp_ls -arm64/qoriq/qoriq_therm_if.m optional pci fdt soc_nxp_ls +arm64/qoriq/qoriq_therm.c optional fdt soc_nxp_ls +arm64/qoriq/qoriq_therm_if.m optional fdt soc_nxp_ls arm64/qoriq/clk/ls1028a_clkgen.c optional clk soc_nxp_ls fdt arm64/qoriq/clk/ls1028a_flexspi_clk.c optional clk soc_nxp_ls fdt arm64/qoriq/clk/ls1046a_clkgen.c optional clk soc_nxp_ls fdt diff --git a/sys/conf/options b/sys/conf/options index e201f8a4bdb7..42045dd11414 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -52,6 +52,7 @@ DDB_CAPTURE_DEFAULTBUFSIZE opt_ddb.h DDB_CAPTURE_MAXBUFSIZE opt_ddb.h DDB_CTF opt_ddb.h DDB_NUMSYM opt_ddb.h +QMAN_CEETM_SUPPORT opt_dpaa.h EARLY_PRINTF opt_global.h FULL_BUF_TRACKING opt_global.h GDB diff --git a/sys/contrib/device-tree/src/arm64/freescale/mono-gateway-dk.dts b/sys/contrib/device-tree/src/arm64/freescale/mono-gateway-dk.dts new file mode 100644 index 000000000000..0f3de66da33b --- /dev/null +++ b/sys/contrib/device-tree/src/arm64/freescale/mono-gateway-dk.dts @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Device Tree for Mono Gateway DK (NXP LS1046A) + * + * Copyright (c) 2026 Mono Technologies Inc. + * Author: Tomaz Zaman + */ + +/dts-v1/; + +#include "fsl-ls1046a.dtsi" + +/ { + model = "Mono Gateway Development Kit"; + compatible = "mono,gateway-dk", "fsl,ls1046a"; + freebsd,dts-version = "6.4"; + + aliases { + serial0 = &duart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + /* + * SFP LED GPIO definitions — referenced by sfp-led-controller via + * the "leds" property on each SFP node. These are NOT children of + * gpio-leds so the gpioled driver won't claim the pins (sfp-led + * module owns them). + */ + led_sfp0_link: sfp0_link { + gpios = <&gpio2 17 GPIO_ACTIVE_HIGH>; + }; + + led_sfp0_activity: sfp0_activity { + gpios = <&gpio2 18 GPIO_ACTIVE_HIGH>; + }; + + led_sfp1_link: sfp1_link { + gpios = <&gpio2 15 GPIO_ACTIVE_HIGH>; + }; + + led_sfp1_activity: sfp1_activity { + gpios = <&gpio2 16 GPIO_ACTIVE_HIGH>; + }; + + sfp_xfi0: sfp-xfi0 { + compatible = "sff,sfp"; + i2c-bus = <&sfp0_i2c>; + tx-disable-gpios = <&gpio2 14 GPIO_ACTIVE_LOW>; + mod-def0-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; + los-gpios = <&gpio2 11 GPIO_ACTIVE_HIGH>; + maximum-power-milliwatt = <3000>; + leds = <&led_sfp0_link>, <&led_sfp0_activity>; + }; + + sfp_xfi1: sfp-xfi1 { + compatible = "sff,sfp"; + i2c-bus = <&sfp1_i2c>; + tx-disable-gpios = <&gpio2 13 GPIO_ACTIVE_LOW>; + mod-def0-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>; + los-gpios = <&gpio2 9 GPIO_ACTIVE_HIGH>; + maximum-power-milliwatt = <3000>; + leds = <&led_sfp1_link>, <&led_sfp1_activity>; + }; + + sfp_led_controller: sfp-led-controller { + compatible = "mono,sfp-led"; + sfp-ports = <&sfp_xfi0>, <&sfp_xfi1>; + }; + +}; + +&soc { + gpio-poweroff { + compatible = "gpio-poweroff"; + gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>; + }; +}; + +&duart0 { + status = "okay"; +}; + +&esdhc { + status = "okay"; + mmc-hs200-1_8v; + bus-width = <4>; +}; + +&i2c0 { + status = "okay"; + + /* Thermal sensors, retimer, clock generator, PWM fan */ + i2c-mux@70 { + compatible = "nxp,pca9545"; + reg = <0x70>; + #address-cells = <1>; + #size-cells = <0>; + + i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + retimer: retimer@18 { + compatible = "ti,ds100df410"; + reg = <0x18>; + status = "okay"; + }; + }; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + + temp-sensor@4c { + compatible = "ti,tmp431"; + reg = <0x4c>; + label = "cpu-board"; + }; + }; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <2>; + + temp-sensor@4c { + compatible = "ti,tmp431"; + reg = <0x4c>; + label = "network-board"; + }; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <3>; + + fan_controller: fan-controller@2e { + compatible = "microchip,emc2305"; + reg = <0x2e>; + #address-cells = <1>; + #size-cells = <0>; + #pwm-cells = <3>; + + fan0: fan@0 { + reg = <0>; + label = "System Fan 1"; + pwms = <&fan_controller 26000 1 1>; + #cooling-cells = <2>; + }; + + fan1: fan@1 { + reg = <1>; + label = "System Fan 2"; + pwms = <&fan_controller 26000 1 1>; + #cooling-cells = <2>; + }; + }; + }; + }; +}; + +&i2c1 { + status = "okay"; + + /* SFP+ I2C mux */ + sfpmux: i2c-mux@70 { + compatible = "nxp,pca9545"; + reg = <0x70>; + #address-cells = <1>; + #size-cells = <0>; + + sfp0_i2c: i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + }; + + sfp1_i2c: i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + }; + }; +}; + +&i2c2 { + status = "okay"; + + i2c-mux@70 { + compatible = "nxp,pca9545"; + reg = <0x70>; + #address-cells = <1>; + #size-cells = <0>; + + i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + power_sensor@40 { + compatible = "ti,ina234"; + reg = <0x40>; + label = "USB Power Delivery"; + #io-channel-cells = <1>; + shunt-resistor = <1000>; + }; + power_sensor@41 { + compatible = "ti,ina234"; + reg = <0x41>; + label = "5V PSU"; + #io-channel-cells = <1>; + shunt-resistor = <1000>; + }; + power_sensor@42 { + compatible = "ti,ina234"; + reg = <0x42>; + label = "1V Core PSU"; + #io-channel-cells = <1>; + shunt-resistor = <1000>; + }; + power_sensor@43 { + compatible = "ti,ina234"; + reg = <0x43>; + label = "1.2V DDR PSU"; + #io-channel-cells = <1>; + shunt-resistor = <5000>; + }; + }; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + + power_sensor@40 { + compatible = "ti,ina234"; + reg = <0x40>; + label = "1.35V SerDes PSU"; + #io-channel-cells = <1>; + shunt-resistor = <5000>; + }; + power_sensor@41 { + compatible = "ti,ina234"; + reg = <0x41>; + label = "1.8V PSU"; + #io-channel-cells = <1>; + shunt-resistor = <5000>; + }; + power_sensor@42 { + compatible = "ti,ina234"; + reg = <0x42>; + label = "2.5V DDR PSU"; + #io-channel-cells = <1>; + shunt-resistor = <5000>; + }; + power_sensor@43 { + compatible = "ti,ina234"; + reg = <0x43>; + label = "3.3V PSU"; + #io-channel-cells = <1>; + shunt-resistor = <1000>; + }; + }; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <2>; + + rtc@53 { + compatible = "nxp,pcf2131"; + reg = <0x53>; + status = "okay"; + }; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <3>; + + lp5810@6c { + compatible = "ti,lp5812"; + reg = <0x6c>; + #address-cells = <1>; + #size-cells = <0>; + + led@0 { + reg = <0>; + label = "status:white"; + led-max-microamp = <25500>; + }; + + led@1 { + reg = <1>; + label = "status:blue"; + led-max-microamp = <25500>; + }; + + led@2 { + reg = <2>; + label = "status:green"; + led-max-microamp = <25500>; + }; + + led@3 { + reg = <3>; + label = "status:red"; + led-max-microamp = <25500>; + }; + }; + }; + }; +}; + +&i2c3 { + status = "okay"; +}; + +#include "fsl-ls1046-post.dtsi" + +/* Override portal bus compatible from "simple-bus" to prevent simplebus + * from claiming these nodes before the DPAA portal drivers attach. + * The portal drivers match on these specific compatible strings. */ +&qportals { + compatible = "fsl,qman-portals"; +}; + +&bportals { + compatible = "fsl,bman-portals"; +}; + +/* CEETM: Channel Egress Enhancement and Traffic Management + * QMan hardware egress QoS — hierarchical traffic shaping with + * sub-portals, LNIs, channels, and class queues. + * LFQID range extracted from Linux SDK DTS for LS1046A. */ +&qman { + qman-ceetm@0 { + compatible = "fsl,qman-ceetm"; + fsl,ceetm-lfqid-range = <0xf00000 0x1000>; + fsl,ceetm-sp-range = <0 16>; + fsl,ceetm-lni-range = <0 8>; + fsl,ceetm-channel-range = <0 32>; + }; +}; + +/* port@82000 (cell-index 2) is reserved for Host Command — disable for OP use. + * Keep 0x3 (IPsec) and 0x4 (WiFi), matching Linux SDK port assignment. */ +&fman0_oh_0x2 { status = "disabled"; }; +&fman0_oh_0x5 { status = "disabled"; }; +&fman0_oh_0x6 { status = "disabled"; }; +&fman0_oh_0x7 { status = "disabled"; }; + +&fman0 { + /* Unused 1G ports */ + ethernet@e0000 { + status = "disabled"; + }; + + ethernet@e4000 { + status = "disabled"; + }; + + ethernet@e6000 { + status = "disabled"; + }; + + /* SGMII 1GbE ports */ + ethernet@e8000 { + phy-handle = <&sgmii_phy0>; + phy-connection-type = "sgmii"; + status = "okay"; + }; + + ethernet@ea000 { + phy-handle = <&sgmii_phy1>; + phy-connection-type = "sgmii"; + status = "okay"; + }; + + ethernet@e2000 { + phy-handle = <&sgmii_phy2>; + phy-connection-type = "sgmii"; + status = "okay"; + }; + + /* 10GbE XFI ports */ + ethernet@f0000 { + sfp = <&sfp_xfi0>; + phy-connection-type = "10gbase-r"; + managed = "in-band-status"; + }; + + ethernet@f2000 { + sfp = <&sfp_xfi1>; + phy-connection-type = "10gbase-r"; + managed = "in-band-status"; + }; + + /* Internal MDIO disabled (no PHYs on this bus) */ + mdio@fc000 { + status = "disabled"; + }; + + /* External MDIO — SGMII PHYs */ + mdio@fd000 { + sgmii_phy0: ethernet-phy@0 { + reg = <0x0>; + }; + + sgmii_phy1: ethernet-phy@1 { + reg = <0x1>; + }; + + sgmii_phy2: ethernet-phy@2 { + reg = <0x2>; + }; + }; +}; + +/* + * Override reserved-memory nodes with fixed physical addresses. + * FreeBSD excludes no-map + reg regions from phys_avail[] at early boot, + * giving QBMan dedicated memory that the VM system never touches. + * Addresses are in DDR range (0x80000000-0xFFFFFFFF), naturally aligned. + */ +&qman_pfdr { + reg = <0 0xc0000000 0 0x2000000>; /* 32MB at 0xC0000000 */ +}; + +&bman_fbpr { + reg = <0 0xc2000000 0 0x1000000>; /* 16MB at 0xC2000000 */ +}; + +&qman_fqd { + reg = <0 0xc3000000 0 0x800000>; /* 8MB at 0xC3000000 */ +}; + +/* + * Board mux/enable GPIOs — configure for 2R (dual-radio / Jody W337). + * 2R-enable: power/mux enable for the PCIe WiFi card + * 3R-enable: disabled (3R = 3-radio card, no PCIe) + * uart-mux: route shared UART to 2R card + */ +&gpio2 { + 2r-enable { + gpio-hog; + gpios = <6 GPIO_ACTIVE_LOW>; + output-high; + line-name = "2r-enable"; + }; + + 3r-enable { + gpio-hog; + gpios = <2 GPIO_ACTIVE_LOW>; + output-low; + line-name = "3r-enable"; + }; + + uart-mux { + gpio-hog; + gpios = <26 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "uart-mux"; + }; +}; + +/* PCIe3: Jody W337 WiFi module (NXP 88W9098) */ +&pcie3 { + status = "okay"; + num-lanes = <1>; +}; + diff --git a/sys/contrib/ncsw/Peripherals/BM/bm.c b/sys/contrib/ncsw/Peripherals/BM/bm.c index 646e38fb9695..09bd8ade79e1 100644 --- a/sys/contrib/ncsw/Peripherals/BM/bm.c +++ b/sys/contrib/ncsw/Peripherals/BM/bm.c @@ -441,7 +441,7 @@ t_Error BmGetRevision(t_Handle h_Bm, t_BmRevisionInfo *p_BmRevisionInfo) static void FreeInitResources(t_Bm *p_Bm) { - if (p_Bm->p_FbprBase) + if (p_Bm->p_FbprBase && !p_Bm->fbprExternal) XX_FreeSmart(p_Bm->p_FbprBase); if (p_Bm->h_Session) XX_IpcFreeSession(p_Bm->h_Session); @@ -491,6 +491,8 @@ t_Handle BM_Config(t_BmParam *p_BmParam) p_Bm->p_BmDriverParams->fbprMemPartitionId = p_BmParam->fbprMemPartitionId; p_Bm->p_BmDriverParams->fbprThreshold = DEFAULT_fbprThreshold; p_Bm->p_BmDriverParams->liodn = p_BmParam->liodn; + p_Bm->p_BmDriverParams->p_FbprBaseExt = p_BmParam->p_FbprBase; + p_Bm->p_BmDriverParams->fbprSizeExt = p_BmParam->fbprSize; } /* build the BM partition IPC address */ @@ -534,26 +536,50 @@ t_Error BM_Init(t_Handle h_Bm) WRITE_UINT32(p_Bm->p_BmRegs->liodnr, (uint16_t)p_Bm->p_BmDriverParams->liodn); /* FBPR memory */ - dsSize = (uint32_t)(p_Bm->p_BmDriverParams->totalNumOfBuffers * (FBPR_ENTRY_SIZE / 8)); - LOG2(dsSize, exp); - if (!POWER_OF_2(dsSize)) (exp++); - dsSize = (uint32_t)(1 << exp); - if (dsSize < (4*KILOBYTE)) + if (p_Bm->p_BmDriverParams->p_FbprBaseExt) { - dsSize = (4*KILOBYTE); + /* Use pre-allocated FBPR memory from caller */ + p_Bm->p_FbprBase = p_Bm->p_BmDriverParams->p_FbprBaseExt; + p_Bm->fbprExternal = TRUE; + dsSize = p_Bm->p_BmDriverParams->fbprSizeExt; LOG2(dsSize, exp); + if (!POWER_OF_2(dsSize)) (exp++); + dsSize = (uint32_t)(1 << exp); } - p_Bm->p_FbprBase = XX_MallocSmart(dsSize, (int)p_Bm->p_BmDriverParams->fbprMemPartitionId, dsSize); - if (!p_Bm->p_FbprBase) + else { - FreeInitResources(p_Bm); - RETURN_ERROR(MAJOR, E_NO_MEMORY, ("FBPR obj!!!")); + dsSize = (uint32_t)(p_Bm->p_BmDriverParams->totalNumOfBuffers * (FBPR_ENTRY_SIZE / 8)); + LOG2(dsSize, exp); + if (!POWER_OF_2(dsSize)) (exp++); + dsSize = (uint32_t)(1 << exp); + if (dsSize < (4*KILOBYTE)) + { + dsSize = (4*KILOBYTE); + LOG2(dsSize, exp); + } + p_Bm->p_FbprBase = XX_MallocSmart(dsSize, (int)p_Bm->p_BmDriverParams->fbprMemPartitionId, dsSize); + if (!p_Bm->p_FbprBase) + { + FreeInitResources(p_Bm); + RETURN_ERROR(MAJOR, E_NO_MEMORY, ("FBPR obj!!!")); + } } phyAddr = XX_VirtToPhys(p_Bm->p_FbprBase); WRITE_UINT32(p_Bm->p_BmRegs->fbpr_bare, ((uint32_t)(phyAddr >> 32) & 0xffff)); WRITE_UINT32(p_Bm->p_BmRegs->fbpr_bar, (uint32_t)phyAddr); WRITE_UINT32(p_Bm->p_BmRegs->fbpr_ar, (exp - 1)); + /* Readback and verify FBPR BAR programming */ + { + uint32_t rb_bare = GET_UINT32(p_Bm->p_BmRegs->fbpr_bare); + uint32_t rb_bar = GET_UINT32(p_Bm->p_BmRegs->fbpr_bar); + uint32_t rb_ar = GET_UINT32(p_Bm->p_BmRegs->fbpr_ar); + XX_Print("BM FBPR: va=%p phys=0x%08x%08x bare=0x%08x bar=0x%08x ar=0x%08x (exp=%d, size=0x%x)\n", + p_Bm->p_FbprBase, + (uint32_t)(phyAddr >> 32), (uint32_t)phyAddr, + rb_bare, rb_bar, rb_ar, exp, dsSize); + } + WRITE_UINT32(p_Bm->p_BmRegs->fbpr_fp_lwit, p_Bm->p_BmDriverParams->fbprThreshold); WRITE_UINT32(p_Bm->p_BmRegs->err_isr, p_Bm->exceptions); WRITE_UINT32(p_Bm->p_BmRegs->err_ier, p_Bm->exceptions); diff --git a/sys/contrib/ncsw/Peripherals/BM/bm.h b/sys/contrib/ncsw/Peripherals/BM/bm.h index 09870cb07156..7fbd30417b41 100644 --- a/sys/contrib/ncsw/Peripherals/BM/bm.h +++ b/sys/contrib/ncsw/Peripherals/BM/bm.h @@ -290,6 +290,8 @@ typedef struct { uint32_t fbprMemPartitionId; uint32_t fbprThreshold; uint16_t liodn; + void *p_FbprBaseExt; /* Pre-allocated FBPR, or NULL */ + uint32_t fbprSizeExt; } t_BmDriverParams; typedef struct { @@ -301,6 +303,7 @@ typedef struct { char moduleName[MODULE_NAME_SIZE]; t_BmRegs *p_BmRegs; void *p_FbprBase; + bool fbprExternal; /* FBPR memory externally allocated */ uint32_t exceptions; t_BmExceptionsCallback *f_Exception; t_Handle h_App; diff --git a/sys/contrib/ncsw/Peripherals/BM/bm_pool.c b/sys/contrib/ncsw/Peripherals/BM/bm_pool.c index c908846760ad..83c0c4b337dd 100644 --- a/sys/contrib/ncsw/Peripherals/BM/bm_pool.c +++ b/sys/contrib/ncsw/Peripherals/BM/bm_pool.c @@ -61,6 +61,7 @@ static void depletion_link(t_BmPool *p_BmPool) NCSW_PLOCK(p_Portal); p_Portal->depletionPoolsTable[p_BmPool->bpid] = p_BmPool; + bman_depletion_set(&p_Portal->pools[0], (uint8_t)p_BmPool->bpid); bm_isr_bscn_mask(p_Portal->p_BmPortalLow, (uint8_t)p_BmPool->bpid, 1); PUNLOCK(p_Portal); } @@ -71,6 +72,7 @@ static void depletion_unlink(t_BmPool *p_BmPool) NCSW_PLOCK(p_Portal); p_Portal->depletionPoolsTable[p_BmPool->bpid] = NULL; + bman_depletion_unset(&p_Portal->pools[0], (uint8_t)p_BmPool->bpid); bm_isr_bscn_mask(p_Portal->p_BmPortalLow, (uint8_t)p_BmPool->bpid, 0); PUNLOCK(p_Portal); } diff --git a/sys/contrib/ncsw/Peripherals/BM/bm_portal.c b/sys/contrib/ncsw/Peripherals/BM/bm_portal.c index 34b48e38b4d8..ff42e31fc20e 100644 --- a/sys/contrib/ncsw/Peripherals/BM/bm_portal.c +++ b/sys/contrib/ncsw/Peripherals/BM/bm_portal.c @@ -381,16 +381,23 @@ t_Error BmPortalRelease(t_Handle h_BmPortal, if (!r) { if (flags & BMAN_RELEASE_FLAG_WAIT) { t_Error ret = wait_rel_start(p_BmPortal, &r, flags); - if (ret) + if (ret) { + printf("BmPortalRelease: wait_rel_start FAILED E_BUSY, avail=%u, " + "rcr.ci=%u, rcr.cursor=%p, rcr.vbit=%u\n", + bm_rcr_get_avail(p_BmPortal->p_BmPortalLow), + p_BmPortal->p_BmPortalLow->rcr.ci, + p_BmPortal->p_BmPortalLow->rcr.cursor, + p_BmPortal->p_BmPortalLow->rcr.vbit); return ret; + } } else return ERROR_CODE(E_BUSY); ASSERT_COND(r != NULL); } r->bpid = bpid; for (i = 0; i < num; i++) { - r->bufs[i].hi = bufs[i].hi; - r->bufs[i].lo = bufs[i].lo; + WRITE_UINT16(r->bufs[i].hi, bufs[i].hi); + WRITE_UINT32(r->bufs[i].lo, bufs[i].lo); } /* Issue the release command and wait for sync if requested. NB: the * commit can't fail, only waiting can. Don't propagate any failure if a @@ -423,8 +430,8 @@ uint8_t BmPortalAcquire(t_Handle h_BmPortal, ASSERT_COND(num <= 8); while (num--) { bufs[num].bpid = bpid; - bufs[num].hi = mcr->acquire.bufs[num].hi; - bufs[num].lo = mcr->acquire.bufs[num].lo; + bufs[num].hi = GET_UINT16(mcr->acquire.bufs[num].hi); + bufs[num].lo = GET_UINT32(mcr->acquire.bufs[num].lo); } PUNLOCK(p_BmPortal); return ret; @@ -495,7 +502,7 @@ t_Handle BM_PORTAL_Config(t_BmPortalParam *p_BmPortalParam) p_BmPortal->h_Bm = p_BmPortalParam->h_Bm; p_BmPortal->p_BmPortalDriverParams->hwExtStructsMemAttr = DEFAULT_memAttr; - bman_depletion_fill(&p_BmPortal->p_BmPortalDriverParams->mask); + bman_depletion_init(&p_BmPortal->p_BmPortalDriverParams->mask); return p_BmPortal; } diff --git a/sys/contrib/ncsw/Peripherals/BM/bman_low.c b/sys/contrib/ncsw/Peripherals/BM/bman_low.c index 818e62df31c0..fb48dc1df3b6 100644 --- a/sys/contrib/ncsw/Peripherals/BM/bman_low.c +++ b/sys/contrib/ncsw/Peripherals/BM/bman_low.c @@ -40,6 +40,7 @@ *//***************************************************************************/ #include #include +#include #include #include "std_ext.h" @@ -55,15 +56,28 @@ /***************************/ /* Cache-inhibited register offsets */ +#if defined(__aarch64__) +/* + * ARM64 Layerscape (LS1046A etc.): CI space is 16KB per portal. + * Register offsets are spread out compared to PowerPC. + * See Linux drivers/soc/fsl/qbman/bman.c for reference. + */ +#define REG_RCR_PI_CINH 0x3000 +#define REG_RCR_CI_CINH 0x3100 +#define REG_RCR_ITR 0x3200 +#define REG_CFG 0x3300 +#define REG_SCN(n) (0x3400 + ((n) << 6)) +#define REG_ISR 0x3e00 +#define BM_ISR_SHIFT 6 /* ISR/IER/ISDR/IIR stride: 64 bytes */ +#else /* PowerPC */ #define REG_RCR_PI_CINH 0x0000 #define REG_RCR_CI_CINH 0x0004 #define REG_RCR_ITR 0x0008 #define REG_CFG 0x0100 #define REG_SCN(n) (0x0200 + ((n) << 2)) #define REG_ISR 0x0e00 -#define REG_IER 0x0e04 -#define REG_ISDR 0x0e08 -#define REG_IIR 0x0e0c +#define BM_ISR_SHIFT 2 /* ISR/IER/ISDR/IIR stride: 4 bytes */ +#endif /* Cache-enabled register offsets */ #define CL_CR 0x0000 @@ -258,9 +272,17 @@ struct bm_rcr_entry *bm_rcr_pend_and_next(struct bm_portal *portal, uint8_t myve void bm_rcr_pci_commit(struct bm_portal *portal, uint8_t myverb) { register struct bm_rcr *rcr = &portal->rcr; + struct bm_rcr_entry *rcursor; ASSERT_COND(rcr->busy); ASSERT_COND(rcr->pmode == e_BmPortalPCI); - rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit); + rcursor = rcr->cursor; + rcursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit); + /* + * ARM64: The portal hardware cannot snoop the CPU cache via CCI-400. + * Explicitly flush the entry from cache to portal CE memory before + * writing PI. On PowerPC the CCF provides hardware coherency. + */ + dcbf_64(rcursor); RCR_INC(rcr); rcr->available--; mb(); @@ -298,7 +320,7 @@ void bm_rcr_pvb_commit(struct bm_portal *portal, uint8_t myverb) struct bm_rcr_entry *rcursor; ASSERT_COND(rcr->busy); ASSERT_COND(rcr->pmode == e_BmPortalPVB); - rmb(); + wmb(); rcursor = rcr->cursor; rcursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit); dcbf_64(rcursor); @@ -487,12 +509,12 @@ void bm_isr_bscn_mask(struct bm_portal *portal, uint8_t bpid, int enable) uint32_t __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n) { - return __bm_in(&portal->addr, REG_ISR + (n << 2)); + return __bm_in(&portal->addr, REG_ISR + (n << BM_ISR_SHIFT)); } void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, uint32_t val) { - __bm_out(&portal->addr, REG_ISR + (n << 2), val); + __bm_out(&portal->addr, REG_ISR + (n << BM_ISR_SHIFT), val); } diff --git a/sys/contrib/ncsw/Peripherals/BM/bman_private.h b/sys/contrib/ncsw/Peripherals/BM/bman_private.h index 972c57cecb0a..2641792e2e62 100644 --- a/sys/contrib/ncsw/Peripherals/BM/bman_private.h +++ b/sys/contrib/ncsw/Peripherals/BM/bman_private.h @@ -55,6 +55,39 @@ * execution reordering with respect to other code/instructions that manipulate * the same cacheline. */ +#if defined(__aarch64__) +/* + * ARM64 cache operations. + * + * Portal CE region is mapped Normal Non-Cacheable (matching Linux's + * MEMREMAP_WC), so stores go directly to hardware. However, dcbf is + * also used on cacheable DMA memory (buffer pools) — there we need + * real cache clean to make CPU writes visible to BMan hardware. + * + * dc cvac (clean by VA to PoC) is the ARM64 equivalent of PPC dcbf. + * On NC addresses it's a harmless no-op; on cached addresses it writes + * dirty lines back to memory. + * + * dc zva MUST NOT be used on portal NC memory — it generates a special + * bus transaction that can hang the interconnect. Use memset instead. + */ +#define lwsync() __asm__ __volatile__ ("dmb ish" : : : "memory") +#define dcbf(addr) do { \ + __asm__ __volatile__ ("dc cvac, %0" : : "r"(addr) : "memory"); \ + __asm__ __volatile__ ("dsb st" : : : "memory"); \ +} while(0) +#define dcbi(p) __asm__ __volatile__ ("dsb sy" : : : "memory") +#define dcbt_ro(addr) do { } while(0) +#define dcbt_rw(addr) do { } while(0) +#define dcbz_64(p) do { memset((p), 0, 64); } while(0) +#define dcbf_64(p) do { \ + __asm__ __volatile__ ("dc cvac, %0" : : "r"(p) : "memory"); \ + __asm__ __volatile__ ("dsb st" : : : "memory"); \ +} while(0) +#define dcbit_ro(p) do { } while(0) + +#else /* PowerPC */ + #define dcbf(addr) \ do { \ __asm__ __volatile__ ("dcbf 0, %0" : : "r" (addr)); \ @@ -127,6 +160,8 @@ #define dcbi(p) dcbf(p) +#endif /* __aarch64__ */ + struct bm_addr { void *addr_ce; /* cache-enabled */ void *addr_ci; /* cache-inhibited */ diff --git a/sys/contrib/ncsw/Peripherals/FM/HC/hc.c b/sys/contrib/ncsw/Peripherals/FM/HC/hc.c index 903d55e4b7e6..0470bbbb7772 100644 --- a/sys/contrib/ncsw/Peripherals/FM/HC/hc.c +++ b/sys/contrib/ncsw/Peripherals/FM/HC/hc.c @@ -228,7 +228,12 @@ static __inline__ t_Error EnQFrm(t_FmHc *p_FmHc, t_DpaaFD *p_FmFd, uint32_t seqN XX_UDelay(100); if (!timeout) + { + intFlags = FmPcdLock(p_FmHc->h_FmPcd); + p_FmHc->enqueued[seqNum] = FALSE; + FmPcdUnlock(p_FmHc->h_FmPcd, intFlags); RETURN_ERROR(MINOR, E_TIMEOUT, ("HC Callback, timeout exceeded")); + } return err; } diff --git a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_cc.c b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_cc.c index a9fd2c1d0368..ba734f9956bd 100644 --- a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_cc.c +++ b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_cc.c @@ -38,6 +38,7 @@ *//***************************************************************************/ #include #include + #include "std_ext.h" #include "error_ext.h" #include "string_ext.h" @@ -51,6 +52,15 @@ #include "fm_cc.h" #include "crc64.h" +#ifdef USE_ENHANCED_EHASH +#include "fm_ehash.h" +extern t_Handle ExternalHashTableSet(t_Handle h_FmPcd, + t_FmPcdHashTableParams *p_Param); +extern t_Error ExternalHashTableModifyMissNextEngine(t_Handle h_HashTbl, + t_FmPcdCcNextEngineParams *p_FmPcdCcNextEngineParams); +extern void copy_td_to_ccbase(void *handle, t_Handle p_CcTreeTmp); +#endif /* USE_ENHANCED_EHASH */ + /****************************************/ /* static functions */ /****************************************/ @@ -344,6 +354,24 @@ static void FillAdOfTypeContLookup(t_Handle h_Ad, /* if (p_AdNewPtr = NULL) --> Done. (case (3)) */ if (p_AdNewPtr) { +#if (DPAA_VERSION >= 11) + if (p_Node->externalHash) + { + /* + * External hash table (CDX): write a zeroed placeholder AD. + * CDX will overwrite this AD with its opcode-format entries + * when it takes over the hash table. Until then, the zero + * AD acts as a "result" type with FQID=0 / NIA=0, which + * causes FMan to enqueue to the default queue. + */ + WRITE_UINT32(p_AdContLookup->ccAdBase, 0); + WRITE_UINT32(p_AdContLookup->matchTblPtr, 0); + WRITE_UINT32(p_AdContLookup->pcAndOffsets, 0); + WRITE_UINT32(p_AdContLookup->gmask, 0); + } + else +#endif /* (DPAA_VERSION >= 11) */ + { /* cases (1) & (2) */ tmpReg32 = 0; tmpReg32 |= FM_PCD_AD_CONT_LOOKUP_TYPE; @@ -371,6 +399,7 @@ static void FillAdOfTypeContLookup(t_Handle h_Ad, MemCpy8((void*)&p_AdContLookup->gmask, p_Node->p_GlblMask, CC_GLBL_MASK_SIZE); + } } } @@ -1071,6 +1100,7 @@ static bool IsCapwapApplSpecific(t_Handle h_Node) } #endif /* FM_CAPWAP_SUPPORT */ +#ifndef USE_ENHANCED_EHASH static t_Error CcUpdateParam( t_Handle h_FmPcd, t_Handle h_PcdParams, t_Handle h_FmPort, t_FmPcdCcKeyAndNextEngineParams *p_CcKeyAndNextEngineParams, @@ -1165,6 +1195,34 @@ static t_Error CcUpdateParam( return E_OK; } +#else /* USE_ENHANCED_EHASH */ +/* + * With enhanced ehash, CC tree entries contain CDX enhanced AD content + * written by copy_td_to_ccbase(). No NCSW internal parameter update + * or recursive tree walking needed — h_CcNode handles are en_exthash_info* + * not t_FmPcdCcNode*, so the internal code would crash. + * Matching Linux SDK fm_cc.c:1209-1221. + */ +static t_Error CcUpdateParam( + t_Handle h_FmPcd, t_Handle h_PcdParams, t_Handle h_FmPort, + t_FmPcdCcKeyAndNextEngineParams *p_CcKeyAndNextEngineParams, + uint16_t numOfEntries, t_Handle h_Ad, bool validate, uint16_t level, + t_Handle h_FmTree, bool modify) +{ + UNUSED(h_FmPcd); + UNUSED(h_PcdParams); + UNUSED(h_FmPort); + UNUSED(p_CcKeyAndNextEngineParams); + UNUSED(numOfEntries); + UNUSED(h_Ad); + UNUSED(validate); + UNUSED(level); + UNUSED(h_FmTree); + UNUSED(modify); + + return E_OK; +} +#endif /* USE_ENHANCED_EHASH */ static ccPrivateInfo_t IcDefineCode(t_FmPcdCcNodeParams *p_CcNodeParam) { @@ -6005,8 +6063,10 @@ t_Handle FM_PCD_CcRootBuild(t_Handle h_FmPcd, t_NetEnvParams netEnvParams; uint8_t lastOne = 0; uint32_t requiredAction = 0; +#ifndef USE_ENHANCED_EHASH t_FmPcdCcNode *p_FmPcdCcNextNode; t_CcNodeInformation ccNodeInfo, *p_CcInformation; +#endif SANITY_CHECK_RETURN_VALUE(h_FmPcd, E_INVALID_HANDLE, NULL); SANITY_CHECK_RETURN_VALUE(p_PcdGroupsParam, E_INVALID_HANDLE, NULL); @@ -6190,6 +6250,7 @@ t_Handle FM_PCD_CcRootBuild(t_Handle h_FmPcd, p_CcTreeTmp = UINT_TO_PTR(p_FmPcdCcTree->ccTreeBaseAddr); +#ifndef USE_ENHANCED_EHASH for (i = 0; i < numOfEntries; i++) { p_KeyAndNextEngineParams = p_Params + i; @@ -6225,10 +6286,59 @@ t_Handle FM_PCD_CcRootBuild(t_Handle h_FmPcd, p_CcInformation->index++; } } +#else /* USE_ENHANCED_EHASH */ + for (i = 0; i < numOfEntries; i++) + { + t_FmPcdCcNextEngineParams *nexteng; + + p_KeyAndNextEngineParams = p_Params + i; + nexteng = &p_KeyAndNextEngineParams->nextEngineParams; + + if (nexteng->nextEngine == e_FM_PCD_CC && + nexteng->params.ccParams.h_CcNode) + { + /* Enhanced external hash table: copy AD into CC tree MURAM slot */ + copy_td_to_ccbase(nexteng->params.ccParams.h_CcNode, p_CcTreeTmp); + } + else + { + /* Non-CC entry (KG/PLCR/DONE): use standard result AD */ + NextStepAd(p_CcTreeTmp, NULL, nexteng, p_FmPcd); + } + + p_CcTreeTmp = PTR_MOVE(p_CcTreeTmp, FM_PCD_CC_AD_ENTRY_SIZE); + + memcpy(&p_FmPcdCcTree->keyAndNextEngineParams[i], + p_KeyAndNextEngineParams, + sizeof(t_FmPcdCcKeyAndNextEngineParams)); + } + + /* + * Fill unused CC tree entries (numOfEntries..15) with a copy of + * entry 0. A zeroed EHASH AD has table_base=0x0 — FMan DMA to + * physical address 0 (not DDR) hangs the task permanently. + * Copying entry 0 gives a valid table_base, int_buf_pool, and + * global_mem_offset. The miss_action starts as DONE (0); + * ExternalHashTableModifyMissNextEngine propagates the correct + * miss action to all copies by scanning for matching table_base_lo. + */ + if (numOfEntries > 0 && numOfEntries < FM_PCD_MAX_NUM_OF_CC_GROUPS) + { + uint8_t *base = (uint8_t *) + UINT_TO_PTR(p_FmPcdCcTree->ccTreeBaseAddr); + for (i = numOfEntries; i < FM_PCD_MAX_NUM_OF_CC_GROUPS; i++) + { + memcpy(base + i * FM_PCD_CC_AD_ENTRY_SIZE, + base, + FM_PCD_CC_AD_ENTRY_SIZE); + } + } +#endif /* USE_ENHANCED_EHASH */ FmPcdIncNetEnvOwners(h_FmPcd, p_FmPcdCcTree->netEnvId); p_CcTreeTmp = UINT_TO_PTR(p_FmPcdCcTree->ccTreeBaseAddr); +#ifndef USE_ENHANCED_EHASH if (!FmPcdLockTryLockAll(p_FmPcd)) { FM_PCD_CcRootDelete(p_FmPcdCcTree); @@ -6259,6 +6369,7 @@ t_Handle FM_PCD_CcRootBuild(t_Handle h_FmPcd, } FmPcdLockUnlockAll(p_FmPcd); +#endif /* USE_ENHANCED_EHASH */ p_FmPcdCcTree->p_Lock = FmPcdAcquireLock(p_FmPcd); if (!p_FmPcdCcTree->p_Lock) { @@ -7112,6 +7223,9 @@ t_Error FM_PCD_MatchTableGetIndexedHashBucket(t_Handle h_CcNode, t_Handle FM_PCD_HashTableSet(t_Handle h_FmPcd, t_FmPcdHashTableParams *p_Param) { +#ifdef USE_ENHANCED_EHASH + return ExternalHashTableSet(h_FmPcd, p_Param); +#else /* !USE_ENHANCED_EHASH */ t_FmPcdCcNode *p_CcNodeHashTbl; t_FmPcdCcNodeParams *p_IndxHashCcNodeParam, *p_ExactMatchCcNodeParam; t_FmPcdCcNode *p_CcNode; @@ -7151,6 +7265,57 @@ t_Handle FM_PCD_HashTableSet(t_Handle h_FmPcd, t_FmPcdHashTableParams *p_Param) ("RMON statistics mode is not supported for hash table")); return NULL; } + + if (p_Param->externalHash) + { + /* + * External hash table: CDX manages the hash buckets in DDR. + * NCSW only allocates a minimal t_FmPcdCcNode with a MURAM AD + * placeholder. CDX will take over the AD and rewrite it with + * CDX opcode entries. + * + * The AD is initially zeroed (safe: result type, enqueue to + * default FQ). Packets reaching this node before CDX takes + * over will be enqueued to the default queue harmlessly. + */ + t_FmPcdCcNode *p_ExtNode; + t_FmPcd *p_FmPcd = (t_FmPcd *)h_FmPcd; + + p_ExtNode = (t_FmPcdCcNode *)XX_Malloc(sizeof(t_FmPcdCcNode)); + if (!p_ExtNode) + { + REPORT_ERROR(MAJOR, E_NO_MEMORY, ("external hash CC node")); + return NULL; + } + memset(p_ExtNode, 0, sizeof(t_FmPcdCcNode)); + + p_ExtNode->externalHash = TRUE; + p_ExtNode->h_FmPcd = h_FmPcd; + p_ExtNode->kgHashShift = p_Param->kgHashShift; + p_ExtNode->statisticsMode = p_Param->statisticsMode; + + /* Allocate MURAM AD for this node */ + p_ExtNode->h_Ad = (t_Handle)FM_MURAM_AllocMem( + p_FmPcd->h_FmMuram, + FM_PCD_CC_AD_ENTRY_SIZE, FM_PCD_CC_AD_TABLE_ALIGN); + if (!p_ExtNode->h_Ad) + { + XX_Free(p_ExtNode); + REPORT_ERROR(MAJOR, E_NO_MEMORY, + ("MURAM allocation for external hash AD")); + return NULL; + } + MemSet8(p_ExtNode->h_Ad, 0, FM_PCD_CC_AD_ENTRY_SIZE); + + p_ExtNode->h_Spinlock = XX_InitSpinlock(); + + INIT_LIST(&p_ExtNode->ccPrevNodesLst); + INIT_LIST(&p_ExtNode->ccTreeIdLst); + INIT_LIST(&p_ExtNode->ccTreesLst); + INIT_LIST(&p_ExtNode->availableStatsLst); + + return (t_Handle)p_ExtNode; + } #endif /* (DPAA_VERSION >= 11) */ p_ExactMatchCcNodeParam = (t_FmPcdCcNodeParams*)XX_Malloc( @@ -7305,10 +7470,14 @@ t_Handle FM_PCD_HashTableSet(t_Handle h_FmPcd, t_FmPcdHashTableParams *p_Param) XX_Free(p_ExactMatchCcNodeParam); return p_CcNodeHashTbl; +#endif /* !USE_ENHANCED_EHASH */ } t_Error FM_PCD_HashTableDelete(t_Handle h_HashTbl) { +#ifdef USE_ENHANCED_EHASH + return (t_Error)-1; /* delete not implemented for enhanced ehash */ +#else t_FmPcdCcNode *p_HashTbl = (t_FmPcdCcNode *)h_HashTbl; t_Handle h_FmPcd; t_Handle *p_HashBuckets, h_MissStatsCounters; @@ -7317,6 +7486,19 @@ t_Error FM_PCD_HashTableDelete(t_Handle h_HashTbl) SANITY_CHECK_RETURN_ERROR(p_HashTbl, E_INVALID_HANDLE); +#if (DPAA_VERSION >= 11) + if (p_HashTbl->externalHash) + { + if (p_HashTbl->h_Ad) + FM_MURAM_FreeMem(FmPcdGetMuramHandle(p_HashTbl->h_FmPcd), + p_HashTbl->h_Ad); + if (p_HashTbl->h_Spinlock) + XX_FreeSpinlock(p_HashTbl->h_Spinlock); + XX_Free(p_HashTbl); + return E_OK; + } +#endif /* (DPAA_VERSION >= 11) */ + /* Store all hash buckets before the hash is freed */ numOfBuckets = p_HashTbl->numOfKeys; @@ -7348,11 +7530,20 @@ t_Error FM_PCD_HashTableDelete(t_Handle h_HashTbl) RETURN_ERROR(MAJOR, err, NO_MSG); return E_OK; +#endif /* !USE_ENHANCED_EHASH */ } t_Error FM_PCD_HashTableAddKey(t_Handle h_HashTbl, uint8_t keySize, t_FmPcdCcKeyParams *p_KeyParams) { +#ifdef USE_ENHANCED_EHASH + /* CDX adds keys directly via ExternalHashTableAddKey (different + * signature). This NCSW dispatch path is not used. */ + UNUSED(h_HashTbl); + UNUSED(keySize); + UNUSED(p_KeyParams); + return E_NOT_SUPPORTED; +#else t_FmPcdCcNode *p_HashTbl = (t_FmPcdCcNode *)h_HashTbl; t_Handle h_HashBucket; uint8_t bucketIndex; @@ -7377,8 +7568,10 @@ t_Error FM_PCD_HashTableAddKey(t_Handle h_HashTbl, uint8_t keySize, return FM_PCD_MatchTableAddKey(h_HashBucket, FM_PCD_LAST_KEY_INDEX, keySize, p_KeyParams); +#endif /* !USE_ENHANCED_EHASH */ } +#ifndef USE_ENHANCED_EHASH t_Error FM_PCD_HashTableRemoveKey(t_Handle h_HashTbl, uint8_t keySize, uint8_t *p_Key) { @@ -7426,11 +7619,16 @@ t_Error FM_PCD_HashTableModifyNextEngine( NULL, p_FmPcdCcNextEngineParams); } +#endif /* !USE_ENHANCED_EHASH */ t_Error FM_PCD_HashTableModifyMissNextEngine( t_Handle h_HashTbl, t_FmPcdCcNextEngineParams *p_FmPcdCcNextEngineParams) { +#ifdef USE_ENHANCED_EHASH + return ExternalHashTableModifyMissNextEngine(h_HashTbl, + p_FmPcdCcNextEngineParams); +#else t_FmPcdCcNode *p_HashTbl = (t_FmPcdCcNode *)h_HashTbl; t_Handle h_HashBucket; uint8_t i; @@ -7483,9 +7681,11 @@ t_Error FM_PCD_HashTableModifyMissNextEngine( } return E_OK; +#endif /* !USE_ENHANCED_EHASH */ } +#ifndef USE_ENHANCED_EHASH t_Error FM_PCD_HashTableGetMissNextEngine( t_Handle h_HashTbl, t_FmPcdCcNextEngineParams *p_FmPcdCcNextEngineParams) @@ -7549,3 +7749,4 @@ t_Error FM_PCD_HashTableGetMissStatistics( return FM_PCD_MatchTableGetMissStatistics(h_HashBucket, p_MissStatistics); } +#endif /* !USE_ENHANCED_EHASH */ diff --git a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_cc.h b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_cc.h index 3456bb565fd7..30bed7c5c44a 100644 --- a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_cc.h +++ b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_cc.h @@ -348,6 +348,11 @@ typedef struct uint32_t shadowAction; uint8_t userSizeOfExtraction; uint8_t userOffset; + +#if (DPAA_VERSION >= 11) + bool externalHash; /* CDX external hash table */ +#endif /* (DPAA_VERSION >= 11) */ + uint8_t kgHashShift; /* used in hash-table */ t_Handle h_Spinlock; diff --git a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_ehash.c b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_ehash.c new file mode 100644 index 000000000000..6974f9f65e23 --- /dev/null +++ b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_ehash.c @@ -0,0 +1,1496 @@ +/* + * fm_ehash.c — FMan enhanced external hash table (kernel-side) + * + * Ported from: + * linux/drivers/net/ethernet/freescale/sdk_fman/Peripherals/FM/Pcd/fm_ehash.c + * + * Implements the ExternalHashTable* API used by CDX for hardware flow + * offload. Manages DDR-based hash tables with CRC64-indexed buckets + * and per-bucket spinlocks. + * + * With USE_ENHANCED_EHASH, FM_PCD_HashTableSet (in fm_cc.c) redirects + * to ExternalHashTableSet here. The returned handle (en_exthash_info*) + * is opaque to all callers (fmlib, FMC, dpa_app, CDX). + * + * The MURAM Action Descriptor is NOT allocated here. copy_td_to_ccbase() + * writes the AD content from info->node into the CC tree's MURAM slot + * during FM_PCD_CcRootBuild, and saves the MURAM address to info->h_Ad. + * ExternalHashTableModifyMissNextEngine then uses info->h_Ad to patch + * the miss action in-place. + * + * Simplifications for initial port: + * - NO_CUMULATIVE_ENTRY: simple linked-list buckets, no compaction + * - No IP reassembly table support (table_type REASSM is accepted + * but reassembly-specific params are ignored) + * + * Copyright 2018 NXP (original Linux SDK code) + * Copyright (c) 2026 Mono Technologies Inc. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "opt_dpaa.h" + +#include +#include +#include +#include +#include + +#include +#include + +/* NCSW public headers (resolved via -I paths in DPAA_COMPILE_CMD) */ +#include "ncsw_ext.h" +#include "error_ext.h" +#include "endian_ext.h" +#include "xx_ext.h" +#include "fm_pcd_ext.h" +#include "fm_muram_ext.h" + +/* NCSW internal headers — needed for t_FmPcd, FmPcdGetMuramHandle, etc. */ +#include "fm_pcd.h" +#include "fm_cc.h" +#include "fm_common.h" + +/* CRC64 from NCSW PCD */ +#include "crc64.h" + +/* CDX fm_ehash.h — entry/bucket structs and opcode constants */ +#include "fm_ehash.h" + +/* Linux endian compat — guarded against LinuxKPI which defines them too */ +#ifndef cpu_to_be16 +#define cpu_to_be16(x) htobe16(x) +#endif +#ifndef cpu_to_be32 +#define cpu_to_be32(x) htobe32(x) +#endif +#ifndef be64_to_cpu +#define be64_to_cpu(x) be64toh(x) +#endif + +/* ---------------------------------------------------------------- + * Direct MURAM accessors — batch-safe volatile stores/loads + * + * MURAM on LS1046A ARM64 requires that multiple stores to the same + * 16-byte region are batched WITHOUT a dsb sy between each store. + * A dsb sy after each store causes all but the last to be lost + * (confirmed empirically: raw batch stores + single dsb work, + * per-store dsb does not). + * + * muram_wr32 does a raw volatile store with bswap, NO barrier. + * muram_rd32 does a raw volatile load with bswap + read barrier. + * muram_barrier() issues a single dsb sy — call ONCE after all + * stores in a batch are complete. + * ---------------------------------------------------------------- */ +#define muram_barrier() __asm __volatile("dsb sy" ::: "memory") + +static __inline void +muram_wr32(volatile uint32_t *addr, uint32_t native_val) +{ + + *addr = __builtin_bswap32(native_val); + /* NO barrier here — caller must call muram_barrier() after batch */ +} + +/* + * muram_wr64 — write two native uint32_t values as a single 64-bit store. + * + * A 64-bit store is a single AXI bus transaction, which the MURAM + * controller handles atomically. This avoids the partial-write + * corruption seen with two separate 32-bit stores when FMan microcode + * concurrently reads the same 16-byte AD region. + * + * On LE ARM64, a 64-bit store writes: + * bits[31:0] → bytes at addr+0 (native_lo, byte-swapped) + * bits[63:32] → bytes at addr+4 (native_hi, byte-swapped) + */ +static __inline void +muram_wr64(volatile uint64_t *addr, uint32_t native_lo, uint32_t native_hi) +{ + + *addr = ((uint64_t)__builtin_bswap32(native_hi) << 32) | + (uint64_t)__builtin_bswap32(native_lo); + /* NO barrier here — caller must call muram_barrier() after batch */ +} + +static __inline uint32_t +muram_rd32(volatile uint32_t *addr) +{ + uint32_t v; + + v = *addr; + __asm __volatile("dsb sy" ::: "memory"); + return (__builtin_bswap32(v)); +} + +/* ---------------------------------------------------------------- + * Internal definitions (supplements fm_ehash.h) + * ---------------------------------------------------------------- */ + +/* Miss action types for en_exthash_node */ +#define EN_EHASH_MISS_ACTION_DROP 3 +#define EN_EHASH_MISS_ACTION_NIA 1 +#define EN_EHASH_MISS_ACTION_ENQUE 2 +#define EN_EHASH_MISS_ACTION_DONE 0 + +/* Internal buffer pool size and table alignment */ +#define EN_INTERNAL_BUFF_POOL_SIZE (256 * 128) +#define EN_EXTHASH_TBL_ALIGNMENT 256 + +/* + * Hardware table_type constants for en_exthash_node word_0 bits [23:20] + * (LE ARM64 C bitfield position). + * The CDX microcode uses table_type to identify the hash table's + * protocol layer. Without a valid table_type, the microcode does + * not recognize the AD as an enhanced hash table entry. + */ +#define EHASH_HW_TABLE_TYPE_L2 (1 << 0) /* 0x1 */ +#define EHASH_HW_TABLE_TYPE_L3 (1 << 1) /* 0x2 */ +#define EHASH_HW_TABLE_TYPE_L4 (1 << 2) /* 0x4 */ +#define EHASH_HW_TABLE_TYPE_REASSM (1 << 3) /* 0x8 */ + +/* + * Convert user-facing table_type enum (from dpa_app/fmlib) to + * hardware table_type constant. Enum values match fm_eh_types.h. + */ +static uint32_t +ehash_hw_table_type(uint32_t user_type) +{ + + switch (user_type & 0xf) { + case 0: /* IPV4_UDP_TABLE */ + case 1: /* IPV4_TCP_TABLE */ + case 2: /* IPV6_UDP_TABLE */ + case 3: /* IPV6_TCP_TABLE */ + case 4: /* ESP_IPV4_TABLE */ + case 5: /* ESP_IPV6_TABLE */ + case 10: /* IPV4_3TUPLE_UDP_TABLE */ + case 11: /* IPV4_3TUPLE_TCP_TABLE */ + case 12: /* IPV6_3TUPLE_UDP_TABLE */ + case 13: /* IPV6_3TUPLE_TCP_TABLE */ + return (EHASH_HW_TABLE_TYPE_L4); + case 6: /* IPV4_MULTICAST_TABLE */ + case 7: /* IPV6_MULTICAST_TABLE */ + return (EHASH_HW_TABLE_TYPE_L3); + case 14: /* IPV4_REASSM_TABLE */ + case 15: /* IPV6_REASSM_TABLE */ + return (EHASH_HW_TABLE_TYPE_REASSM); + default: /* PPPOE_RELAY_TABLE, ETHERNET_TABLE, etc. */ + return (EHASH_HW_TABLE_TYPE_L2); + } +} + +/* Flags for en_exthash_info.flags */ +#define TIMESTAMP_EN (1 << 0) +#define STATS_EN (1 << 1) + +/* + * en_exthash_node — Action Descriptor for enhanced external hash table. + * Written to MURAM (via copy_td_to_ccbase); read by FMan CDX microcode. + * + * The Linux SDK (fm_ehash.h line 563) defines this with C bitfield unions + * "assuming a LE GPP core". On LE ARM64, the first declared struct field + * occupies the LSB. muram_wr32/muram_wr64 byte-swap the native value + * so FMan (always BE) reads the correct numeric value. + * + * CONFIRMED: LE ARM64 C bitfield positions are correct. Attempting + * PPC/BE positions (build #194) causes FMan exception 5 (crash). + * + * Non-EXCLUDE_FMAN_IPR_OFFLOAD layout (active on LS1046A): + * + * word_0 — LE ARM64 C bitfield positions (first field at LSB): + * bits 7:0 = table_base_hi (8) — upper bits of 40-bit table addr + * bits 15:8 = ipv4_ad_offset (8) — IPR reassembly AD offset (0) + * bits 18:16 = hash_bytes_offset (3) — CRC64 shift parameter + * bit 19 = reserved (1) + * bits 23:20 = table_type (4) — L2/L3/L4/REASSM identifier + * bits 29:24 = key_size (6) + * bits 31:30 = miss_action_type (2) + * + * word_1 — LE ARM64 C bitfield positions: + * bits 3:0 = hash_mask_bits (4) + * bits 15:4 = global_mem_offset (12) + * bits 31:16 = int_buf_pool_addr (16) + * + * word_2: nia or fqid depending on miss_action_type + */ +struct en_exthash_node { + uint32_t word_0; + uint32_t table_base_lo; + uint32_t word_1; + uint32_t word_2; /* nia or fqid */ +} __attribute__((packed)); + +/* word_0 field construction — LE ARM64 C bitfield positions. + * + * On LE ARM64, C bitfields place the first declared field at bit 0 (LSB). + * muram_wr32/muram_wr64 byte-swap the native value for MURAM storage. + * + * Non-EXCLUDE_FMAN_IPR_OFFLOAD variant (LS1046A). */ +#define EHASH_W0_TABLE_BASE_HI(v) ((uint32_t)(v) & 0xFF) /* bits 7:0 */ +#define EHASH_W0_IPV4_AD_OFF(v) (((uint32_t)(v) & 0xFF) << 8) /* bits 15:8 */ +#define EHASH_W0_HASH_BYTES_OFF(v) (((uint32_t)(v) & 0x7) << 16) /* bits 18:16 */ +#define EHASH_W0_TABLE_TYPE(v) (((uint32_t)(v) & 0xF) << 20) /* bits 23:20 */ +#define EHASH_W0_KEY_SIZE(v) (((uint32_t)(v) & 0x3F) << 24) /* bits 29:24 */ +#define EHASH_W0_MISS_ACTION(v) (((uint32_t)(v) & 0x3) << 30) /* bits 31:30 */ +#define EHASH_W0_MISS_ACTION_MASK (0x3U << 30) /* bits 31:30 */ + +/* word_1 field construction — LE ARM64 C bitfield positions. + * Non-EXCLUDE variant: hash_mask_bits at LSB, then + * global_mem_offset, then int_buf_pool_addr at MSB. */ +#define EHASH_W1_HASH_MASK_BITS(v) ((uint32_t)(v) & 0xF) /* bits 3:0 */ +#define EHASH_W1_GLOBAL_MEM_OFF(v) (((uint32_t)(v) & 0xFFF) << 4) /* bits 15:4 */ +#define EHASH_W1_INT_BUF_POOL(v) (((uint32_t)(v) & 0xFFFF) << 16) /* bits 31:16 */ + +/* + * en_exthash_info — Per-table software state. + * Returned as opaque t_Handle to callers. + */ +struct en_exthash_info { + uint32_t flags; + void *table_base; /* DDR bucket array */ + void **pSpinlock; /* per-bucket spinlock array */ + void *h_Ad; /* MURAM AD pointer (set by copy_td_to_ccbase) */ + struct en_exthash_node node; /* local copy of table descriptor */ + uint32_t tablesize; + uint32_t hashmask; + uint32_t keysize; + uint32_t hashshift; + uint32_t dataMemId; + uint32_t dataLiodnOffset; + uint32_t num_keys; + uint32_t max_collisions; + void *pcd; /* PCD handle */ + uint32_t table_type; /* user-facing type (L2/L3/L4/REASSM) */ + struct ip_reassembly_params *ip_reassem_info; /* MURAM VA, REASSM only */ +}; + +MALLOC_DEFINE(M_EHASH, "ehash", "CDX external hash table"); + +/* ---------------------------------------------------------------- + * Internal buffer pool management — allocated once in MURAM + * ---------------------------------------------------------------- */ + +static void *ehash_int_muram_ptr; /* virtual address */ +static uint32_t ehash_int_muram_area; /* MURAM offset >> 8 */ +static void *ehash_first_table_base;/* DDR base of first table */ + +/* Global MURAM data area — dscp_vlanpcp_map + SEC_failure_stats. + * Points to pool + EN_INTERNAL_BUFF_POOL_SIZE. Initialized in + * ehash_init_muram_pool(). */ +static en_exthash_global_mem *ehash_global_mem; + +/* Reassembly table tracking — set during ExternalHashTableSet() + * for IPV4_REASSM_TABLE (14) and IPV6_REASSM_TABLE (15). */ +#define EHASH_IPV4_REASSM_TABLE 14 +#define EHASH_IPV6_REASSM_TABLE 15 + +static struct en_exthash_info *ehash_ipv4_reassly_tbl; +static struct en_exthash_info *ehash_ipv6_reassly_tbl; + +/* ---------------------------------------------------------------- + * External timestamp slots — separate MURAM allocation + * + * Timestamps MUST be in their own MURAM region, NOT in the global + * data area. The global data area (at int_buf_pool + POOL_SIZE) + * contains en_exthash_global_mem (dscp_vlanpcp_map + SEC_failure_stats) + * which the CDX microcode reads/writes at runtime. + * + * Linux SDK allocates timestamps separately in FM_PCD_Init() + * (fm_pcd.c: FM_MURAM_AllocMem for MAX_EXT_TS_TIMERS * EXT_TS_SIZE). + * ---------------------------------------------------------------- */ + +#define MAX_EXT_TS_TIMERS 4 + +static uint32_t *ehash_ts_virt; /* virtual ptr to slot[0] */ +static uint64_t ehash_ts_phys; /* physical addr of slot[0] */ +static uint64_t ehash_muram_phys_base; /* MURAM physical base */ + +/* + * Allocate the internal buffer pool + global data area in MURAM, + * plus a SEPARATE timestamp region. + * Called once on first ExternalHashTableSet. + */ +static int +ehash_init_muram_pool(t_Handle h_FmPcd) +{ + t_FmPcd *p_FmPcd = (t_FmPcd *)h_FmPcd; + t_Handle h_FmMuram; + uint32_t total_size; + uint32_t muram_offset; + uint32_t ts_size; + + if (ehash_int_muram_ptr != NULL) + return (0); + + h_FmMuram = FmPcdGetMuramHandle(h_FmPcd); + if (h_FmMuram == NULL) + return (-1); + + /* + * Layout of the pool+global allocation: + * [0 .. EN_INTERNAL_BUFF_POOL_SIZE-1] internal buffer pool + * [EN_INTERNAL_BUFF_POOL_SIZE .. +256] global data area + * (en_exthash_global_mem: dscp_vlanpcp_map + SEC_failure_stats) + * + * Timestamps are allocated SEPARATELY below. + */ + total_size = EN_INTERNAL_BUFF_POOL_SIZE + 256; + + ehash_int_muram_ptr = FM_MURAM_AllocMem(h_FmMuram, + total_size, EN_EXTHASH_TBL_ALIGNMENT); + if (ehash_int_muram_ptr == NULL) { + printf("fm_ehash: MURAM alloc failed (%u bytes)\n", + total_size); + return (-1); + } + IOMemSet32(ehash_int_muram_ptr, 0, total_size); + + /* Compute compressed MURAM offset (physical offset / 256) */ + muram_offset = (uint32_t)(XX_VirtToPhys(ehash_int_muram_ptr) - + p_FmPcd->physicalMuramBase); + if (muram_offset & 0xff) { + printf("fm_ehash: MURAM pool not 256-aligned (0x%x)\n", + muram_offset); + FM_MURAM_FreeMem(h_FmMuram, ehash_int_muram_ptr); + ehash_int_muram_ptr = NULL; + return (-1); + } + + ehash_int_muram_area = muram_offset >> 8; + + ehash_muram_phys_base = p_FmPcd->physicalMuramBase; + + printf("fm_ehash: MURAM pool at %p, offset 0x%x (compressed 0x%x)\n", + ehash_int_muram_ptr, muram_offset, ehash_int_muram_area); + ehash_global_mem = (en_exthash_global_mem *)((uint8_t *)ehash_int_muram_ptr + + EN_INTERNAL_BUFF_POOL_SIZE); + + printf("fm_ehash: global data at pool+0x%x (en_exthash_global_mem)\n", + EN_INTERNAL_BUFF_POOL_SIZE); + + /* + * Allocate timestamp slots as a SEPARATE MURAM region. + * Linux SDK does this in FM_PCD_Init() via FM_MURAM_AllocMem. + * These must NOT overlap the global data area. + */ + ts_size = MAX_EXT_TS_TIMERS * sizeof(uint32_t); + ehash_ts_virt = FM_MURAM_AllocMem(h_FmMuram, + ts_size, sizeof(uint32_t)); + if (ehash_ts_virt == NULL) { + printf("fm_ehash: MURAM timestamp alloc failed (%u bytes)\n", + ts_size); + FM_MURAM_FreeMem(h_FmMuram, ehash_int_muram_ptr); + ehash_int_muram_ptr = NULL; + return (-1); + } + IOMemSet32(ehash_ts_virt, 0, ts_size); + ehash_ts_phys = XX_VirtToPhys(ehash_ts_virt); + + printf("fm_ehash: timestamp slots at %p, phys 0x%jx " + "(MURAM offset 0x%jx)\n", + ehash_ts_virt, (uintmax_t)ehash_ts_phys, + (uintmax_t)(ehash_ts_phys - ehash_muram_phys_base)); + + return (0); +} + +/* ---------------------------------------------------------------- + * CRC64-based hash bucket indexing + * + * Ported from sdk_fman fm_cc.c:get_indexed_hash_bucket() + * ---------------------------------------------------------------- */ + +static void +get_indexed_hash_bucket(uint8_t key_size, uint8_t *key_ptr, + uint8_t crc_shift, uint16_t mask, uint16_t *bucket_index) +{ + uint64_t crc64; + + crc64 = crc64_init(); + crc64 = crc64_compute(key_ptr, key_size, crc64); + crc64 >>= ((6 - crc_shift) << 3); + *bucket_index = (uint16_t)crc64 & mask; +} + +/* ---------------------------------------------------------------- + * Bucket chain search (NO_CUMULATIVE_ENTRY path) + * ---------------------------------------------------------------- */ + +static struct en_exthash_tbl_entry * +find_entry_in_bucket(struct en_exthash_tbl_entry *entry, + uint8_t *key, uint32_t size) +{ + + while (entry != NULL) { + if (memcmp(key, &entry->hashentry.key[0], size) == 0) + return (entry); + entry = entry->next; + } + return (NULL); +} + +/* ---------------------------------------------------------------- + * Cleanup helper + * ---------------------------------------------------------------- */ + +static void +delete_ehash_info(struct en_exthash_info *info) +{ + uint32_t ii; + + if (info == NULL) + return; + + if (info->pSpinlock != NULL) { + for (ii = 0; ii <= info->hashmask; ii++) { + if (*(info->pSpinlock + ii) != NULL) + XX_FreeSpinlock(*(info->pSpinlock + ii)); + else + break; + } + free(info->pSpinlock, M_EHASH); + } + + if (info->table_base != NULL) + contigfree(info->table_base, info->tablesize, M_EHASH); + + /* + * Do NOT free h_Ad — it points into the CC tree's MURAM slot + * (set by copy_td_to_ccbase). The CC tree owns that MURAM. + */ + + free(info, M_EHASH); +} + + +/* ================================================================ + * Public API + * ================================================================ */ + +/* + * ExternalHashTableSet — Create an enhanced external hash table. + * + * Called from FM_PCD_HashTableSet (via USE_ENHANCED_EHASH redirect). + * Allocates DDR bucket array and per-bucket spinlocks. Builds the + * AD node data in info->node but does NOT write it to MURAM. + * copy_td_to_ccbase() (called during FM_PCD_CcRootBuild) writes + * the AD into the CC tree's MURAM entry and sets info->h_Ad. + * + * Returns opaque handle on success, NULL on failure. + */ +t_Handle +ExternalHashTableSet(t_Handle h_FmPcd, t_FmPcdHashTableParams *p_Param) +{ + struct en_exthash_info *info; + uint32_t ii; + struct en_exthash_node *node; + uint64_t tblphysaddr; + uint32_t num_bits; + + if (ehash_init_muram_pool(h_FmPcd) != 0) + return (NULL); + + info = malloc(sizeof(*info), M_EHASH, M_WAITOK | M_ZERO); + if (info == NULL) { + REPORT_ERROR(MAJOR, E_NO_MEMORY, + ("en_exthash_info allocation")); + return (NULL); + } + + info->hashmask = p_Param->hashResMask; + info->pcd = h_FmPcd; + + /* + * Validate hash mask: must be (2^N - 1) where N <= 15. + */ + if (info->hashmask > 0x7fff) { + REPORT_ERROR(MAJOR, E_INVALID_VALUE, + ("unsupported hash mask value")); + goto err_ret; + } + + num_bits = 0; + for (ii = 0; ii < 15; ii++) { + if ((info->hashmask + 1) & (1U << ii)) + break; + } + num_bits = ii; + + if ((1U << num_bits) != (info->hashmask + 1)) { + REPORT_ERROR(MAJOR, E_INVALID_VALUE, + ("hash mask not power-of-2 minus 1")); + goto err_ret; + } + + /* Allocate per-bucket spinlocks (CPU-only, no DMA) */ + info->pSpinlock = malloc( + sizeof(t_Handle) * (info->hashmask + 1), + M_EHASH, M_WAITOK | M_ZERO); + if (info->pSpinlock == NULL) { + REPORT_ERROR(MAJOR, E_NO_MEMORY, ("spinlock array")); + goto err_ret; + } + + for (ii = 0; ii <= info->hashmask; ii++) { + *(info->pSpinlock + ii) = XX_InitSpinlock(); + if (*(info->pSpinlock + ii) == NULL) { + REPORT_ERROR(MAJOR, E_NO_MEMORY, + ("bucket spinlock")); + goto err_ret; + } + } + + /* + * Allocate DDR hash table (bucket array). + * Must be physically contiguous — FMan microcode accesses + * buckets via table_base_phys + index * sizeof(bucket). + * Use contigmalloc directly instead of the shared NCSW + * MallocSmart pool, which is sized for dTSEC buffers only. + */ + info->tablesize = (uint32_t)(sizeof(struct en_exthash_bucket) * + (info->hashmask + 1)); + info->table_base = contigmalloc(info->tablesize, M_EHASH, + M_WAITOK | M_ZERO, 0, (vm_paddr_t)0xFFFFFFFF, + EN_EXTHASH_TBL_ALIGNMENT, 0); + if (info->table_base == NULL) { + REPORT_ERROR(MAJOR, E_NO_MEMORY, + ("hash table buckets")); + goto err_ret; + } + + /* Save first table base for diagnostic DDR bucket verification */ + if (ehash_first_table_base == NULL) + ehash_first_table_base = info->table_base; + + /* Fill table parameters */ + info->keysize = p_Param->matchKeySize; + info->hashshift = p_Param->hashShift; + info->flags |= TIMESTAMP_EN; + if (p_Param->statisticsMode) + info->flags |= STATS_EN; + + /* Build AD node in local copy — NOT written to MURAM yet. + * copy_td_to_ccbase() writes this into the CC tree MURAM slot. */ + node = &info->node; + memset(node, 0, sizeof(*node)); + + tblphysaddr = XX_VirtToPhys(info->table_base); + + { + uint32_t miss_action; + uint32_t hw_table_type; + + /* Convert user-facing table_type enum to hardware constant. + * p_Param->table_type is populated by dpa_app via fmlib. */ + hw_table_type = ehash_hw_table_type(p_Param->table_type); + + switch (p_Param->ccNextEngineParamsForMiss.nextEngine) { + case e_FM_PCD_KG: { + t_FmPcdCcNextKgParams *kgparams = + &p_Param->ccNextEngineParamsForMiss.params.kgParams; + + node->word_2 = NIA_ENG_KG | NIA_KG_DIRECT | + NIA_KG_CC_EN; + if (kgparams->overrideFqid) + node->word_2 |= kgparams->newFqid; + node->word_2 |= + FmPcdKgGetSchemeId(kgparams->h_DirectScheme); + miss_action = EN_EHASH_MISS_ACTION_NIA; + break; + } + case e_FM_PCD_DONE: { + t_FmPcdCcNextEnqueueParams *enqparams = + &p_Param->ccNextEngineParamsForMiss.params + .enqueueParams; + + if (enqparams->overrideFqid) { + node->word_2 = enqparams->newFqid; + miss_action = EN_EHASH_MISS_ACTION_ENQUE; + } else { + miss_action = EN_EHASH_MISS_ACTION_DONE; + } + break; + } + default: + miss_action = EN_EHASH_MISS_ACTION_DROP; + break; + } + + /* Build word_0 — non-EXCLUDE layout matching CDX microcode. + * ipv4_ad_offset = 0 (no IPR reassembly support). */ + node->word_0 = + EHASH_W0_TABLE_BASE_HI((tblphysaddr >> 32) & 0xff) | + EHASH_W0_IPV4_AD_OFF(0) | + EHASH_W0_HASH_BYTES_OFF(info->hashshift) | + EHASH_W0_TABLE_TYPE(hw_table_type) | + EHASH_W0_KEY_SIZE(info->keysize) | + EHASH_W0_MISS_ACTION(miss_action); + node->table_base_lo = tblphysaddr & 0xffffffff; + node->word_1 = + EHASH_W1_HASH_MASK_BITS(num_bits) | + EHASH_W1_GLOBAL_MEM_OFF(EN_INTERNAL_BUFF_POOL_SIZE >> 8) | + EHASH_W1_INT_BUF_POOL(ehash_int_muram_area); + } + + /* + * No MURAM AD allocation here. info->h_Ad stays NULL until + * copy_td_to_ccbase() writes the AD into the CC tree's MURAM + * slot during FM_PCD_CcRootBuild. + */ + info->h_Ad = NULL; + info->table_type = p_Param->table_type; + + /* + * Track reassembly tables for ExternalHashSetReasslyPool(). + * For REASSM tables, word_2 holds the MURAM offset (>>8) of + * ip_reassembly_params. Compute the VA so we can write pool + * configuration fields later. + */ + if (p_Param->table_type == EHASH_IPV4_REASSM_TABLE || + p_Param->table_type == EHASH_IPV6_REASSM_TABLE) { + t_FmPcd *p_FmPcd = (t_FmPcd *)h_FmPcd; + uint64_t reassm_phys; + + if (node->word_2 != 0) { + reassm_phys = p_FmPcd->physicalMuramBase + + ((uint64_t)node->word_2 << 8); + info->ip_reassem_info = + (struct ip_reassembly_params *)XX_PhysToVirt( + reassm_phys); + printf("fm_ehash: REASSM table type %u, " + "ip_reassem_info at %p (MURAM off 0x%x)\n", + p_Param->table_type, info->ip_reassem_info, + node->word_2 << 8); + } + + if (p_Param->table_type == EHASH_IPV4_REASSM_TABLE) + ehash_ipv4_reassly_tbl = info; + else + ehash_ipv6_reassly_tbl = info; + } + + return ((t_Handle)info); + +err_ret: + delete_ehash_info(info); + return (NULL); +} + +/* + * copy_td_to_ccbase — Write enhanced hash AD into CC tree MURAM entry. + * + * Called from FM_PCD_CcRootBuild (USE_ENHANCED_EHASH path) for each + * CC tree entry that points to a hash table. Copies the 16-byte AD + * from info->node to the CC tree's MURAM slot using big-endian + * accessors, and saves the MURAM address to info->h_Ad. + * + * After this call, FMan reads the CC tree entry and finds CDX enhanced + * hash format. ExternalHashTableModifyMissNextEngine can later patch + * the miss action via info->h_Ad. + * + * No IPR reassembly support (void return). + */ +void +copy_td_to_ccbase(void *handle, t_Handle p_CcTreeTmp) +{ + struct en_exthash_info *info = (struct en_exthash_info *)handle; + volatile uint32_t *p = (volatile uint32_t *)p_CcTreeTmp; + + struct en_exthash_node *ptr = &info->node; + + /* + * Write enhanced hash AD into CC tree MURAM slot. + * + * Enhanced hash node layout (16 bytes, all BE in MURAM): + * p[0] = word_0: flags, miss_action, int_buf_pool, etc. + * p[1] = table_base_lo: DDR hash table physical address (low 32) + * p[2] = word_1: global_mem_offset, hash_mask, key_size + * p[3] = word_2: miss FQID (set to 0 here, patched by ModifyMiss) + * + * The CDX microcode reads this format for CC lookup. + * Standard Result ADs do NOT work — CDX microcode ignores + * the standard FQID field and uses KG baseFqid instead + * (confirmed build #207: all frames discarded with QMan + * Invalid Enqueue State for FQID 65882). + */ + { + uint32_t v0, v1, v2, v3; + int retry; + + for (retry = 0; retry < 4; retry++) { + muram_wr32(&p[2], ptr->word_1); /* offset 8 */ + muram_wr32(&p[1], ptr->table_base_lo); /* offset 4 */ + muram_wr32(&p[3], ptr->word_2); /* offset 12 */ + muram_wr32(&p[0], ptr->word_0); /* offset 0 */ + muram_barrier(); + + v0 = muram_rd32(&p[0]); + v1 = muram_rd32(&p[1]); + v2 = muram_rd32(&p[2]); + v3 = muram_rd32(&p[3]); + if (v0 == ptr->word_0 && + v1 == ptr->table_base_lo && + v2 == ptr->word_1 && + v3 == ptr->word_2) + break; + + printf("fm_ehash: copy_td VERIFY FAIL attempt %d " + "AD=%p [%08x %08x %08x %08x] != " + "[%08x %08x %08x %08x]\n", + retry + 1, p, v0, v1, v2, v3, + ptr->word_0, ptr->table_base_lo, + ptr->word_1, ptr->word_2); + } + } + + /* Save MURAM address for AD — used by ModifyMissNextEngine later */ + info->h_Ad = (void *)p; +} + +/* + * ExternalHashTableAllocEntry — Allocate a new hash table entry. + */ +void * +ExternalHashTableAllocEntry(void *h_HashTbl) +{ + struct en_exthash_info *info; + void *entry; + + info = (struct en_exthash_info *)h_HashTbl; + if (info == NULL) + return (NULL); + + entry = XX_MallocSmart(sizeof(struct en_exthash_tbl_entry), + 0, EN_EHASH_ENTRY_ALIGN); + if (entry != NULL) + memset(entry, 0, sizeof(struct en_exthash_tbl_entry)); + else + REPORT_ERROR(MAJOR, E_NO_MEMORY, ("hash table entry")); + + return (entry); +} + +/* + * ExternalHashTableEntryFree — Free a hash table entry. + */ +void +ExternalHashTableEntryFree(void *entry) +{ + + if (entry != NULL) + XX_FreeSmart(entry); +} + +/* + * ExternalHashTableAddKey — Add an entry to the hash table. + * + * Returns the bucket index on success, -1 on failure. + */ +int +ExternalHashTableAddKey(void *h_HashTbl, uint8_t keySize, void *tbl_entry) +{ + struct en_exthash_info *info; + struct en_exthash_tbl_entry *first_entry; + struct en_exthash_tbl_entry *new_entry; + uint16_t index; + struct en_exthash_bucket *bucket; + t_Handle *h_Spinlock; + uint32_t intFlags; + int retval; + uint64_t phyaddr; + + SANITY_CHECK_RETURN_ERROR(h_HashTbl, E_INVALID_HANDLE); + SANITY_CHECK_RETURN_ERROR(tbl_entry, E_NULL_POINTER); + + new_entry = (struct en_exthash_tbl_entry *)tbl_entry; + info = (struct en_exthash_info *)h_HashTbl; + + if (info->table_base == NULL || info->pSpinlock == NULL) { + REPORT_ERROR(MAJOR, E_INVALID_HANDLE, + ("hash table not initialized (table_base=%p pSpinlock=%p)", + info->table_base, info->pSpinlock)); + return (-1); + } + + get_indexed_hash_bucket(keySize, &new_entry->hashentry.key[0], + info->hashshift, (uint16_t)info->hashmask, &index); + + bucket = (struct en_exthash_bucket *)info->table_base + index; + + h_Spinlock = *(info->pSpinlock + index); + intFlags = XX_LockIntrSpinlock(h_Spinlock); + + retval = index; + phyaddr = bucket->h; + + if (phyaddr) { + first_entry = XX_PhysToVirt(SwapUint64(phyaddr)); + if (find_entry_in_bucket(first_entry, + &new_entry->hashentry.key[0], keySize)) { + REPORT_ERROR(MAJOR, E_ALREADY_EXISTS, + ("hash table entry")); + retval = -1; + goto func_ret; + } + first_entry->prev = new_entry; + } else { + first_entry = NULL; + } + + /* Link new entry at head of bucket chain */ + new_entry->next = first_entry; + + /* Fill hardware next-entry pointer (byte-swapped physical addr) */ + phyaddr = SwapUint64(phyaddr); + new_entry->hashentry.next_entry_hi = + cpu_to_be16((phyaddr >> 32) & 0xffff); + new_entry->hashentry.next_entry_lo = + cpu_to_be32(phyaddr & 0xffffffff); + + /* Update bucket head to point to new entry */ + phyaddr = XX_VirtToPhys(new_entry); + bucket->h = SwapUint64(phyaddr); + +func_ret: + XX_UnlockIntrSpinlock(h_Spinlock, intFlags); + return (retval); +} + +/* + * ExternalHashTableDeleteKey — Remove an entry from the hash table. + * + * Returns 0 on success, -1 on failure. + */ +int +ExternalHashTableDeleteKey(void *h_HashTbl, uint16_t index, void *tbl_entry) +{ + struct en_exthash_info *info; + struct en_exthash_bucket *bucket; + struct en_exthash_tbl_entry *entry; + struct en_exthash_tbl_entry *temp_entry; + t_Handle *h_Spinlock; + uint32_t intFlags; + uint64_t phyaddr; + uint64_t update_entry; + + if (h_HashTbl == NULL || tbl_entry == NULL) + return (-1); + + info = (struct en_exthash_info *)h_HashTbl; + + if (info->table_base == NULL || info->pSpinlock == NULL) { + REPORT_ERROR(MAJOR, E_INVALID_HANDLE, + ("hash table not initialized in DeleteKey")); + return (-1); + } + + if (index > info->hashmask) { + REPORT_ERROR(MAJOR, E_INVALID_VALUE, + ("DeleteKey index %u exceeds hashmask %u", + (unsigned)index, (unsigned)info->hashmask)); + return (-1); + } + + bucket = (struct en_exthash_bucket *)info->table_base + index; + entry = (struct en_exthash_tbl_entry *)tbl_entry; + + h_Spinlock = *(info->pSpinlock + index); + intFlags = XX_LockIntrSpinlock(h_Spinlock); + + /* Mark entry as invalid for microcode */ + update_entry = SwapUint64(entry->hashentry.next_entry); + SET_INVALID_ENTRY_64BIT(update_entry); + entry->hashentry.next_entry = SwapUint64(update_entry); + + if (entry->prev != NULL) { + /* Not at head — unlink from middle/end */ + temp_entry = entry->prev; + temp_entry->next = entry->next; + if (entry->next != NULL) + entry->next->prev = temp_entry; + /* Update hardware next-entry pointer */ + update_entry = entry->hashentry.next_entry & ~0xffffULL; + temp_entry->hashentry.next_entry = + update_entry | temp_entry->hashentry.flags; + } else { + /* At head — update bucket pointer */ + if (entry->next != NULL) { + phyaddr = XX_VirtToPhys(entry->next); + bucket->h = SwapUint64(phyaddr); + entry->next->prev = NULL; + } else { + /* Last entry in bucket — clear to zero (empty) */ + bucket->h = 0; + } + } + + XX_UnlockIntrSpinlock(h_Spinlock, intFlags); + + if (FmPcdHcSync(info->pcd)) { + printf("fm_ehash: FmPcdHcSync failed in DeleteKey\n"); + return (-1); + } + + return (0); +} + +/* + * ExternalHashTableEntryGetStatsAndTS — Read entry stats and timestamp. + */ +int +ExternalHashTableEntryGetStatsAndTS(void *tbl_entry, + struct en_tbl_entry_stats *stats) +{ + struct en_exthash_tbl_entry *hash_node; + struct en_ehash_entry *entry; + uint16_t flags; + + hash_node = (struct en_exthash_tbl_entry *)tbl_entry; + if (hash_node == NULL) + return (-1); + + entry = &hash_node->hashentry; + flags = cpu_to_be16(entry->flags); + stats->flags = 0; + + if (GET_TIMESTAMP_ENABLE(flags)) { + stats->flags |= TIMESTAMP_VALID; + stats->timestamp = cpu_to_be32(entry->timestamp); + } + + if (GET_STATS_ENABLE(flags)) { + stats->flags |= STATS_VALID; + stats->pkts = be64_to_cpu(entry->packet_count); + stats->bytes = be64_to_cpu(entry->packet_bytes); + } + + return (0); +} + +/* + * ExternalHashTableModifyMissNextEngine — Change the miss action. + * + * Matches Linux NXP SDK fm_ehash.c:1090-1181 with ARM64 MURAM + * write adaptations: + * + * 1. Re-read current AD from MURAM (not cached info->node) — the + * CDX microcode or NCSW PCD setup may have modified the AD + * since copy_td_to_ccbase() wrote it. + * + * 2. Two-phase write: set miss action to DROP first, then update + * word_2 (NIA/FQID) and word_0 (real miss action). Prevents + * microcode from seeing a partially-updated AD. + * + * 3. All 4 words written in batch — partial writes corrupt + * unwritten words (LS1046A MURAM / Normal-NC ARM64 behavior). + * + * 4. KG always sets NIA_KG_CC_EN (Linux fm_ehash.c:1115). + * + * 5. PLCR uses newRelativeProfileId directly (Linux fm_ehash.c:1151). + */ +t_Error +ExternalHashTableModifyMissNextEngine(t_Handle h_HashTbl, + t_FmPcdCcNextEngineParams *p_FmPcdCcNextEngineParams) +{ + struct en_exthash_info *info; + struct en_exthash_node *ad; + uint32_t w0, w3; + + info = (struct en_exthash_info *)h_HashTbl; + if (info->h_Ad == NULL) { + printf("fm_ehash: ModifyMissNextEngine: h_Ad is NULL " + "(copy_td_to_ccbase not called?)\n"); + return (E_INVALID_STATE); + } + + ad = (struct en_exthash_node *)info->h_Ad; + + /* + * Read current word_0 and word_2 from MURAM. + * Match Linux fm_ehash.c:1097-1098. + */ + w0 = GET_UINT32(ad->word_0); + w3 = GET_UINT32(ad->word_2); + + /* + * Phase 1: Set miss action to DROP while we update. + * Match Linux fm_ehash.c:1100-1102. + * Write ONLY word_0 — do NOT touch other words. + */ + WRITE_UINT32(ad->word_0, (w0 & ~EHASH_W0_MISS_ACTION_MASK) | + EHASH_W0_MISS_ACTION(EN_EHASH_MISS_ACTION_DROP)); + + /* + * Phase 2: Build new miss action. + * Match Linux fm_ehash.c:1105-1164. + */ + switch (p_FmPcdCcNextEngineParams->nextEngine) { + case e_FM_PCD_KG: { + t_FmPcdCcNextKgParams *kgparams = + &p_FmPcdCcNextEngineParams->params.kgParams; + + w3 = NIA_ENG_KG | NIA_KG_DIRECT | NIA_KG_CC_EN; + w3 |= FmPcdKgGetSchemeId(kgparams->h_DirectScheme); + w0 = (w0 & ~EHASH_W0_MISS_ACTION_MASK) | + EHASH_W0_MISS_ACTION(EN_EHASH_MISS_ACTION_NIA); + break; + } + case e_FM_PCD_PLCR: { + t_FmPcdCcNextPlcrParams *plcrparams = + &p_FmPcdCcNextEngineParams->params.plcrParams; + + if (plcrparams->sharedProfile) + w3 = NIA_ENG_PLCR | NIA_PLCR_ABSOLUTE | + plcrparams->newRelativeProfileId; + else + w3 = NIA_ENG_PLCR | + plcrparams->newRelativeProfileId; + w0 = (w0 & ~EHASH_W0_MISS_ACTION_MASK) | + EHASH_W0_MISS_ACTION(EN_EHASH_MISS_ACTION_NIA); + break; + } + case e_FM_PCD_DONE: { + t_FmPcdCcNextEnqueueParams *enqparams = + &p_FmPcdCcNextEngineParams->params.enqueueParams; + + if (enqparams->overrideFqid) { + w3 = enqparams->newFqid; + w0 = (w0 & ~EHASH_W0_MISS_ACTION_MASK) | + EHASH_W0_MISS_ACTION(EN_EHASH_MISS_ACTION_ENQUE); + } else { + w0 = (w0 & ~EHASH_W0_MISS_ACTION_MASK) | + EHASH_W0_MISS_ACTION(EN_EHASH_MISS_ACTION_DONE); + } + break; + } + default: + printf("fm_ehash: ModifyMiss: unhandled engine %d\n", + p_FmPcdCcNextEngineParams->nextEngine); + w0 = (w0 & ~EHASH_W0_MISS_ACTION_MASK) | + EHASH_W0_MISS_ACTION(EN_EHASH_MISS_ACTION_DROP); + break; + } + + /* + * Phase 3: Write ONLY word_2 then word_0 to MURAM. + * Match Linux fm_ehash.c:1166-1167 exactly. + * Use WRITE_UINT32 (NCSW macro = out32rb + dsb per store). + * Do NOT touch table_base_lo or word_1. + */ + WRITE_UINT32(ad->word_2, w3); + WRITE_UINT32(ad->word_0, w0); + + return (E_OK); +} + +/* + * ExternalHashTableFmPcdHcSync — Flush PCD Host Command channel. + */ +int +ExternalHashTableFmPcdHcSync(void *h_HashTbl) +{ + struct en_exthash_info *info; + + info = (struct en_exthash_info *)h_HashTbl; + if (FmPcdHcSync(info->pcd)) { + printf("fm_ehash: FmPcdHcSync failed\n"); + return (-1); + } + return (0); +} + +/* + * ExternalHashTableDelete — Destroy a table and free all resources. + */ +void +ExternalHashTableDelete(t_Handle h_HashTbl) +{ + + if (h_HashTbl != NULL) + delete_ehash_info((struct en_exthash_info *)h_HashTbl); +} + +/* ================================================================ + * Global MURAM features — read/write the global data area at + * pool + EN_INTERNAL_BUFF_POOL_SIZE (en_exthash_global_mem). + * + * All fields are big-endian in MURAM. Use GET_UINT32/WRITE_UINT32 + * for 32-bit fields, GET_UINT8/WRITE_UINT8 for byte fields. + * ================================================================ */ + +int32_t +ExternalHashGetSECfailureStats(en_SEC_failure_stats *stats) +{ + volatile en_SEC_failure_stats *src; + + if (stats == NULL) + return (-1); + if (ehash_global_mem == NULL) { + memset(stats, 0, sizeof(*stats)); + return (-1); + } + + src = &ehash_global_mem->SEC_failure_stats; + + stats->icv_failures = GET_UINT32(src->icv_failures); + stats->hw_errs = GET_UINT32(src->hw_errs); + stats->CCM_AAD_size_errs = GET_UINT32(src->CCM_AAD_size_errs); + stats->anti_replay_late_errs = GET_UINT32(src->anti_replay_late_errs); + stats->anti_replay_replay_errs = GET_UINT32(src->anti_replay_replay_errs); + stats->seq_num_overflows = GET_UINT32(src->seq_num_overflows); + stats->DMA_errs = GET_UINT32(src->DMA_errs); + stats->DECO_watchdog_timer_timedout_errs = GET_UINT32(src->DECO_watchdog_timer_timedout_errs); + stats->input_frame_read_errs = GET_UINT32(src->input_frame_read_errs); + stats->protocol_format_errs = GET_UINT32(src->protocol_format_errs); + stats->ipsec_ttl_zero_errs = GET_UINT32(src->ipsec_ttl_zero_errs); + stats->ipsec_pad_chk_failures = GET_UINT32(src->ipsec_pad_chk_failures); + stats->output_frame_length_rollover_errs = GET_UINT32(src->output_frame_length_rollover_errs); + stats->tbl_buff_too_small_errs = GET_UINT32(src->tbl_buff_too_small_errs); + stats->tbl_buff_pool_depletion_errs = GET_UINT32(src->tbl_buff_pool_depletion_errs); + stats->output_frame_too_large_errs = GET_UINT32(src->output_frame_too_large_errs); + stats->cmpnd_frame_write_errs = GET_UINT32(src->cmpnd_frame_write_errs); + stats->buff_too_small_errs = GET_UINT32(src->buff_too_small_errs); + stats->buff_pool_depletion_errs = GET_UINT32(src->buff_pool_depletion_errs); + stats->output_frame_write_errs = GET_UINT32(src->output_frame_write_errs); + stats->cmpnd_frame_read_errs = GET_UINT32(src->cmpnd_frame_read_errs); + stats->prehdr_read_errs = GET_UINT32(src->prehdr_read_errs); + stats->other_errs = GET_UINT32(src->other_errs); + + return (0); +} + +int32_t +ExternalHashResetSECfailureStats(void) +{ + volatile en_SEC_failure_stats *dst; + + if (ehash_global_mem == NULL) + return (-1); + + dst = &ehash_global_mem->SEC_failure_stats; + + WRITE_UINT32(dst->icv_failures, 0); + WRITE_UINT32(dst->hw_errs, 0); + WRITE_UINT32(dst->CCM_AAD_size_errs, 0); + WRITE_UINT32(dst->anti_replay_late_errs, 0); + WRITE_UINT32(dst->anti_replay_replay_errs, 0); + WRITE_UINT32(dst->seq_num_overflows, 0); + WRITE_UINT32(dst->DMA_errs, 0); + WRITE_UINT32(dst->DECO_watchdog_timer_timedout_errs, 0); + WRITE_UINT32(dst->input_frame_read_errs, 0); + WRITE_UINT32(dst->protocol_format_errs, 0); + WRITE_UINT32(dst->ipsec_ttl_zero_errs, 0); + WRITE_UINT32(dst->ipsec_pad_chk_failures, 0); + WRITE_UINT32(dst->output_frame_length_rollover_errs, 0); + WRITE_UINT32(dst->tbl_buff_too_small_errs, 0); + WRITE_UINT32(dst->tbl_buff_pool_depletion_errs, 0); + WRITE_UINT32(dst->output_frame_too_large_errs, 0); + WRITE_UINT32(dst->cmpnd_frame_write_errs, 0); + WRITE_UINT32(dst->buff_too_small_errs, 0); + WRITE_UINT32(dst->buff_pool_depletion_errs, 0); + WRITE_UINT32(dst->output_frame_write_errs, 0); + WRITE_UINT32(dst->cmpnd_frame_read_errs, 0); + WRITE_UINT32(dst->prehdr_read_errs, 0); + WRITE_UINT32(dst->other_errs, 0); + + return (0); +} + +int32_t +ExternalHashSetDscpVlanpcpMapCfg(en_dscp_vlanpcp_map_cfg *map) +{ + volatile uint8_t *dst; + int ii; + + if (map == NULL) + return (-1); + if (ehash_global_mem == NULL) + return (-1); + + dst = ehash_global_mem->dscp_vlanpcp_map.dscp_vlanpcp; + for (ii = 0; ii <= MAX_VLAN_PCP; ii++) + WRITE_UINT8(dst[ii], map->dscp_vlanpcp[ii]); + + return (0); +} + +int32_t +ExternalHashGetDscpVlanpcpMapCfg(en_dscp_vlanpcp_map_cfg *map) +{ + volatile uint8_t *src; + int ii; + + if (map == NULL) + return (-1); + if (ehash_global_mem == NULL) { + memset(map, 0, sizeof(*map)); + return (-1); + } + + src = ehash_global_mem->dscp_vlanpcp_map.dscp_vlanpcp; + for (ii = 0; ii <= MAX_VLAN_PCP; ii++) + map->dscp_vlanpcp[ii] = GET_UINT8(src[ii]); + + return (0); +} + +/* + * ExternalHashReasslyTableExists — Check if a reassembly table exists. + * + * Returns 1 if the specified reassembly table (IPv4 or IPv6) was + * created during PCD setup, 0 otherwise. Used by the CDX module + * to skip resource allocation when reassembly is not configured. + */ +int +ExternalHashReasslyTableExists(uint32_t type) +{ + struct en_exthash_info *tbl_info; + + switch (type) { + case EHASH_IPV4_REASSM_TABLE: + tbl_info = ehash_ipv4_reassly_tbl; + break; + case EHASH_IPV6_REASSM_TABLE: + tbl_info = ehash_ipv6_reassly_tbl; + break; + default: + return (0); + } + + return (tbl_info != NULL && tbl_info->ip_reassem_info != NULL); +} + +/* + * ExternalHashSetReasslyPool — Configure reassembly buffer pools. + * + * Called from CDX module (cdx_reassm_freebsd.c) to program BMan pool + * IDs, buffer sizes, timeout, and TX-confirm FQIDs into the MURAM + * ip_reassembly_params structure for IPv4/IPv6 reassembly tables. + * + * The ip_reassem_info pointer is set during ExternalHashTableSet() + * when a REASSM table is created. If no REASSM table exists (e.g. + * dpa_app PCD XML doesn't include reassembly), this returns -1. + */ +int +ExternalHashSetReasslyPool(uint32_t type, uint32_t ctx_bpid, + uint32_t ctx_bpsize, uint32_t frag_bpid, uint32_t frag_size, + uint32_t txc_fqid, uint32_t ipr_timer_freq) +{ + struct en_exthash_info *tbl_info; + volatile struct ip_reassembly_params *params; + uint32_t timeout; + + switch (type) { + case EHASH_IPV4_REASSM_TABLE: + tbl_info = ehash_ipv4_reassly_tbl; + break; + case EHASH_IPV6_REASSM_TABLE: + tbl_info = ehash_ipv6_reassly_tbl; + break; + default: + printf("fm_ehash: SetReasslyPool: invalid type %u\n", type); + return (-1); + } + + if (tbl_info == NULL || tbl_info->ip_reassem_info == NULL) + return (-1); + + params = (volatile struct ip_reassembly_params *)tbl_info->ip_reassem_info; + + printf("fm_ehash: SetReasslyPool type %u: ctx_bpid=%u frag_bpid=%u " + "timer_freq=%u txc_fqid=%u\n", + type, ctx_bpid, frag_bpid, ipr_timer_freq, txc_fqid); + + WRITE_UINT32(params->reassem_bpid, ctx_bpid); + WRITE_UINT32(params->reassem_bsize, ctx_bpsize); + WRITE_UINT32(params->frag_bpid, frag_bpid); + WRITE_UINT32(params->frag_bsize, frag_size); + WRITE_UINT32(params->txc_fqid, txc_fqid); + + /* Convert timeout from absolute to timer-frequency-relative. + * The existing timeout_val was set by the PCD configuration + * in absolute units; divide by timer frequency. */ + if (ipr_timer_freq != 0) { + timeout = GET_UINT32(params->timeout_val); + WRITE_UINT32(params->timeout_val, timeout / ipr_timer_freq); + } + + return (0); +} + +void +ipr_update_timestamp(void) +{ + + /* Timestamp updates handled by cdx_ehash_update_timestamp() */ +} + +/* + * cdx_ehash_update_timestamp — Write current time to MURAM timestamp slot. + * + * Called from the CDX timer thread (via dpa_update_timestamp) on every + * tick. FMan microcode reads these slots and copies the raw value into + * each hash entry's timestamp field on packet match (MURAM-to-MURAM). + * + * NO byte-swap: Linux SDK's FM_PCD_UpdateExtTimeStamp does a raw store. + * CDX software reads the timestamp back with the same raw access, so + * both sides see the same byte pattern. + */ +void +cdx_ehash_update_timestamp(uint32_t id, uint32_t value) +{ + + if (ehash_ts_virt == NULL || id >= MAX_EXT_TS_TIMERS) + return; + *(volatile uint32_t *)&ehash_ts_virt[id] = value; + __asm __volatile("dsb sy" ::: "memory"); +} + +/* + * cdx_ehash_get_timestamp_addr — Return MURAM physical offset for a + * timestamp slot. + */ +uint32_t +cdx_ehash_get_timestamp_addr(uint32_t id) +{ + + if (ehash_ts_virt == NULL || id >= MAX_EXT_TS_TIMERS) + return (0); + return ((uint32_t)(ehash_ts_phys + id * sizeof(uint32_t) - + ehash_muram_phys_base)); +} + +/* + * ehash_dump_muram_diag — Dump MURAM state for stall diagnosis. + * + * Prints global data area, timestamp slots, and first internal + * buffer pool slot. Safe to call from callout context. + */ +static void +ehash_dump_muram_diag(void) +{ + uint8_t *global_data; + volatile uint32_t *pool32; + + if (ehash_int_muram_ptr == NULL) { + printf("fm_ehash: diag: pool not initialized\n"); + return; + } + + /* Global data area: at pool + EN_INTERNAL_BUFF_POOL_SIZE */ + global_data = (uint8_t *)ehash_int_muram_ptr + + EN_INTERNAL_BUFF_POOL_SIZE; + printf("fm_ehash: global_data (dscp_vlanpcp_map): " + "%02x %02x %02x %02x %02x %02x %02x %02x\n", + global_data[0], global_data[1], global_data[2], global_data[3], + global_data[4], global_data[5], global_data[6], global_data[7]); + /* SEC failure stats — 23 × uint32_t starting at global_data+8. + * CDX microcode writes these in BE (via WRITE_UINT32), so read + * with GET_UINT32 for correct byte order. */ + { + static const char *stat_names[] = { + "icv_fail", "hw_err", "ccm_aad", "ar_late", + "ar_replay", "seq_ovfl", "dma_err", "deco_wdt", + "in_rd_err", "proto_fmt", "ttl_zero", "pad_chk", + "out_len_roll", "tbl_buf_small", "tbl_pool_depl", + "out_too_large", "cmpnd_wr", "buf_small", + "pool_depl", "out_wr", "cmpnd_rd", + "prehdr_rd", "other" + }; + volatile uint32_t *sp = + (volatile uint32_t *)(global_data + 8); + int si, any = 0; + for (si = 0; si < 23; si++) { + uint32_t v = GET_UINT32(sp[si]); + if (v != 0) { + printf("fm_ehash: SEC stat[%d] %s = %u\n", + si, stat_names[si], v); + any = 1; + } + } + if (!any) + printf("fm_ehash: SEC stats: all zero\n"); + } + + /* Timestamp slots (separate allocation) */ + if (ehash_ts_virt != NULL) + printf("fm_ehash: ts[0..3]: %08x %08x %08x %08x " + "(raw, no swap)\n", + *(volatile uint32_t *)&ehash_ts_virt[0], + *(volatile uint32_t *)&ehash_ts_virt[1], + *(volatile uint32_t *)&ehash_ts_virt[2], + *(volatile uint32_t *)&ehash_ts_virt[3]); + else + printf("fm_ehash: ts: not allocated\n"); + + /* Internal buffer pool — first 4 slots (each 256 bytes). + * Read with GET_UINT32 since microcode writes in BE. */ + pool32 = (volatile uint32_t *)ehash_int_muram_ptr; + { + int slot; + for (slot = 0; slot < 4; slot++) { + int base = slot * 64; /* 256B / 4B per word */ + printf("fm_ehash: pool[%d]: %08x %08x %08x %08x\n", + slot, + GET_UINT32(pool32[base + 0]), + GET_UINT32(pool32[base + 1]), + GET_UINT32(pool32[base + 2]), + GET_UINT32(pool32[base + 3])); + } + } + + /* DDR hash table bucket[0] verification */ + if (ehash_first_table_base != NULL) { + volatile uint64_t *bkt = + (volatile uint64_t *)ehash_first_table_base; + printf("fm_ehash: DDR bucket[0] (cached): " + "%016lx %016lx\n", bkt[0], bkt[1]); + /* Writeback + invalidate to force re-read from DDR */ + cpu_dcache_wbinv_range((vm_offset_t)ehash_first_table_base, 64); + printf("fm_ehash: DDR bucket[0] (after wbinv): " + "%016lx %016lx\n", bkt[0], bkt[1]); + } +} + +/* ================================================================ + * Debug + * ================================================================ */ + +static void +EhashTableWalk(void *h_HashTbl) +{ + struct en_exthash_info *info; + struct en_exthash_tbl_entry *entry; + struct en_exthash_bucket *bucket; + uint32_t ii; + uint32_t num_entries = 0; + uint32_t max_collisions = 0; + + info = (struct en_exthash_info *)h_HashTbl; + if (info == NULL) + return; + + bucket = (struct en_exthash_bucket *)info->table_base; + for (ii = 0; ii <= info->hashmask; ii++) { + if (bucket->h != 0) { + uint32_t bucket_entries = 0; + + entry = XX_PhysToVirt(SwapUint64(bucket->h)); + while (entry != NULL) { + bucket_entries++; + entry = entry->next; + } + num_entries += bucket_entries; + if (bucket_entries > max_collisions) + max_collisions = bucket_entries; + } + bucket++; + } + + printf("fm_ehash: table %p — %u entries, max collisions %u, " + "%u buckets\n", info, num_entries, max_collisions, + info->hashmask + 1); +} diff --git a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_kg.c b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_kg.c index f183d2f9ee05..08ffdc821b31 100644 --- a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_kg.c +++ b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_kg.c @@ -1035,6 +1035,12 @@ static t_Error BuildSchemeRegs(t_FmPcdKgScheme *p_Scheme, memset(swDefaults, 0, NUM_OF_SW_DEFAULTS*sizeof(t_FmPcdKgExtractDflt)); memset(p_SchemeRegs, 0, sizeof(struct fman_kg_scheme_regs)); + /* Default: disable policer profile generation. PLCR and + * CC-with-plcrNext cases below overwrite kgse_ppc when PGEN + * is actually needed. Without this, kgse_ppc=0 leaves + * PP_NO_GEN clear → implicit PGEN with profile 0. */ + p_SchemeRegs->kgse_ppc = KG_SCH_PP_NO_GEN; + if (p_SchemeParams->netEnvParams.numOfDistinctionUnits > FM_PCD_MAX_NUM_OF_DISTINCTION_UNITS) RETURN_ERROR(MAJOR, E_INVALID_VALUE, ("numOfDistinctionUnits should not exceed %d", FM_PCD_MAX_NUM_OF_DISTINCTION_UNITS)); @@ -1250,7 +1256,13 @@ static t_Error BuildSchemeRegs(t_FmPcdKgScheme *p_Scheme, } p_SchemeRegs->kgse_mode = tmpReg; - p_SchemeRegs->kgse_mv = p_Scheme->matchVector; + /* SCHEME_ALWAYS_DIRECT (0xFFFFFFFF) means "match all frames" in NCSW + * software, but FMan KG hardware interprets the MV register literally: + * (classification_vector & kgse_mv) == kgse_mv. With 0xFFFFFFFF, all + * 32 classification bits must be set — no frame can ever match. + * Write 0 instead: the hardware treats MV=0 as "always match." */ + p_SchemeRegs->kgse_mv = (p_Scheme->matchVector == SCHEME_ALWAYS_DIRECT) + ? 0 : p_Scheme->matchVector; #if (DPAA_VERSION >= 11) if (p_SchemeParams->overrideStorageProfile) @@ -1569,6 +1581,17 @@ static t_Error BuildSchemeRegs(t_FmPcdKgScheme *p_Scheme, generic = FALSE; } } + /* + * CDX/ASK flow offload requires the port ID to always be + * present as the first byte of every KG key output. The + * PCD XML only adds it to + * extractedOrs (for FQID modification), but the CDX hash + * table key (union dpa_key) expects portid at byte 0. + * Force KG_SCH_KN_PORT_ID into the known-fields set so + * the hardware extracts it as known-field ID 0 (bit 31, + * highest priority → first in key output). + */ + knownTmp |= KG_SCH_KN_PORT_ID; p_SchemeRegs->kgse_ekfc = knownTmp; selectTmp = 0; diff --git a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_pcd.h b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_pcd.h index 27ec9c5bf672..d1d4acd009c3 100644 --- a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_pcd.h +++ b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_pcd.h @@ -437,6 +437,7 @@ t_Handle PrsConfig(t_FmPcd *p_FmPcd,t_FmPcdParams *p_FmPcdParams); t_Error PrsInit(t_FmPcd *p_FmPcd); void PrsEnable(t_FmPcd *p_FmPcd); void PrsDisable(t_FmPcd *p_FmPcd); +int PrsIsEnabled(t_FmPcd *p_FmPcd); void PrsFree(t_FmPcd *p_FmPcd ); t_Error PrsIncludePortInStatistics(t_FmPcd *p_FmPcd, uint8_t hardwarePortId, bool include); @@ -540,4 +541,6 @@ static __inline__ void FmPcdLockUnlock(t_FmPcdLock *p_Lock) } +t_Handle FM_PCD_GetHcPort(t_Handle h_FmPcd); + #endif /* __FM_PCD_H */ diff --git a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_plcr.c b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_plcr.c index f23d8b8168b2..db687ad82201 100644 --- a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_plcr.c +++ b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_plcr.c @@ -158,6 +158,14 @@ static t_Error SetProfileNia(t_FmPcd *p_FmPcd, e_FmPcdEngine nextEngine, u_FmPcd RETURN_ERROR(MAJOR, E_INVALID_STATE, ("Invalid profile ")); nia |= NIA_ENG_PLCR | NIA_PLCR_ABSOLUTE | absoluteProfileId; break; + case e_FM_PCD_PRS: + /* Post-policer: re-enter PCD pipeline via KG engine */ + nia |= 0x00480200; + break; + case e_FM_PCD_CC: + /* Post-policer: coarse classifier engine */ + nia |= 0x26; + break; default: RETURN_ERROR(MAJOR, E_INVALID_SELECTION, NO_MSG); } diff --git a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_replic.c b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_replic.c index ee82f73014b6..6bf8cc096947 100644 --- a/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_replic.c +++ b/sys/contrib/ncsw/Peripherals/FM/Pcd/fm_replic.c @@ -144,7 +144,7 @@ static t_FmPcdFrmReplicMember *GetAvailableMember(t_FmPcdFrmReplicGroup *p_Repli if (!LIST_IsEmpty(&p_ReplicGroup->availableMembersList)) { - p_Next = LIST_FIRST(&p_ReplicGroup->availableMembersList); + p_Next = NCSW_LIST_FIRST(&p_ReplicGroup->availableMembersList); p_ReplicMember = LIST_OBJECT(p_Next, t_FmPcdFrmReplicMember, node); ASSERT_COND(p_ReplicMember); LIST_DelAndInit(p_Next); @@ -643,7 +643,7 @@ void FrmReplicGroupUpdateOwner(t_Handle h_ReplicGroup, } } -t_Error FrmReplicGroupTryLock(t_Handle h_ReplicGroup) +static t_Error FrmReplicGroupTryLock(t_Handle h_ReplicGroup) { t_FmPcdFrmReplicGroup *p_ReplicGroup = (t_FmPcdFrmReplicGroup *)h_ReplicGroup; @@ -655,7 +655,7 @@ t_Error FrmReplicGroupTryLock(t_Handle h_ReplicGroup) return ERROR_CODE(E_BUSY); } -void FrmReplicGroupUnlock(t_Handle h_ReplicGroup) +static void FrmReplicGroupUnlock(t_Handle h_ReplicGroup) { t_FmPcdFrmReplicGroup *p_ReplicGroup = (t_FmPcdFrmReplicGroup *)h_ReplicGroup; diff --git a/sys/contrib/ncsw/Peripherals/FM/Port/fm_port.c b/sys/contrib/ncsw/Peripherals/FM/Port/fm_port.c index 79419eab453f..f46cb2c840b0 100644 --- a/sys/contrib/ncsw/Peripherals/FM/Port/fm_port.c +++ b/sys/contrib/ncsw/Peripherals/FM/Port/fm_port.c @@ -36,6 +36,9 @@ @Description FM driver routines implementation. *//***************************************************************************/ +#ifdef __FreeBSD__ +#endif + #include "error_ext.h" #include "std_ext.h" #include "string_ext.h" @@ -1349,7 +1352,7 @@ static t_Error SetPcd(t_FmPort *p_FmPort, t_FmPortPcdParams *p_PcdParams) } /* if CC is used directly after BMI */ - if ((p_PcdParams->pcdSupport == e_FM_PORT_PCD_SUPPORT_CC_ONLY) + if (p_PcdParams->pcdSupport == e_FM_PORT_PCD_SUPPORT_CC_ONLY #ifdef FM_CAPWAP_SUPPORT || (p_PcdParams->pcdSupport == e_FM_PORT_PCD_SUPPORT_CC_AND_KG) || (p_PcdParams->pcdSupport == e_FM_PORT_PCD_SUPPORT_CC_AND_KG_AND_PLCR) @@ -5271,6 +5274,7 @@ t_Error FM_PORT_SetPCD(t_Handle h_FmPort, t_FmPortPcdParams *p_PcdParam) RETURN_ERROR(MAJOR, err, NO_MSG); } + /* Diagnostic: dump all BMI NIA registers after SetPCD */ RELEASE_LOCK(p_FmPort->lock); return err; @@ -5668,7 +5672,7 @@ static t_Error FmPortConfigAutoResForDeepSleepSupport1(t_FmPort *p_FmPort) return E_OK; } -t_FmPortDsarTablesSizes* FM_PORT_GetDsarTablesMaxSizes(t_Handle h_FmPortRx) +static t_FmPortDsarTablesSizes* FM_PORT_GetDsarTablesMaxSizes(t_Handle h_FmPortRx) { t_FmPort *p_FmPort = (t_FmPort *)h_FmPortRx; return p_FmPort->deepSleepVars.autoResMaxSizes; diff --git a/sys/contrib/ncsw/Peripherals/FM/fm_ncsw.c b/sys/contrib/ncsw/Peripherals/FM/fm_ncsw.c index 884bc45edd3b..33915989b1eb 100644 --- a/sys/contrib/ncsw/Peripherals/FM/fm_ncsw.c +++ b/sys/contrib/ncsw/Peripherals/FM/fm_ncsw.c @@ -88,8 +88,11 @@ static bool IsFmanCtrlCodeLoaded(t_Fm *p_Fm) static t_Error CheckFmParameters(t_Fm *p_Fm) { +#ifndef __aarch64__ + /* On ARM64, U-Boot preloads FMan microcode into IRAM; reuse it */ if (IsFmanCtrlCodeLoaded(p_Fm) && !p_Fm->resetOnInit) RETURN_ERROR(MAJOR, E_INVALID_VALUE, ("Old FMan CTRL code is loaded; FM must be reset!")); +#endif #if (DPAA_VERSION < 11) if (!p_Fm->p_FmDriverParam->dma_axi_dbg_num_of_beats || (p_Fm->p_FmDriverParam->dma_axi_dbg_num_of_beats > DMA_MODE_MAX_AXI_DBG_NUM_OF_BEATS)) @@ -3434,7 +3437,7 @@ t_Handle FM_Config(t_FmParams *p_FmParam) #ifdef FM_AID_MODE_NO_TNUM_SW005 if (p_Fm->p_FmStateStruct->revInfo.majorRev >= 6) - p_Fm->p_FmDriverParam->dma_aid_mode = e_FM_DMA_AID_OUT_PORT_ID; + p_Fm->p_FmDriverParam->dma_aid_mode = E_FMAN_DMA_AID_OUT_PORT_ID; #endif /* FM_AID_MODE_NO_TNUM_SW005 */ #ifndef FM_QMI_NO_DEQ_OPTIONS_SUPPORT if (p_Fm->p_FmStateStruct->revInfo.majorRev != 4) @@ -3568,12 +3571,15 @@ t_Error FM_Init(t_Handle h_Fm) if (!p_Fm->resetOnInit) /* Skip operations done in errata workaround */ { #endif /* FM_UCODE_NOT_RESET_ERRATA_BUGZILLA6173 */ - /* Load FMan-Controller code to IRAM */ - - ClearIRam(p_Fm); + /* Load FMan-Controller code to IRAM (skip if no firmware provided, + e.g. when U-Boot has already loaded microcode) */ + if (p_Fm->firmware.p_Code) + { + ClearIRam(p_Fm); - if (p_Fm->firmware.p_Code && (LoadFmanCtrlCode(p_Fm) != E_OK)) - RETURN_ERROR(MAJOR, E_INVALID_STATE, NO_MSG); + if (LoadFmanCtrlCode(p_Fm) != E_OK) + RETURN_ERROR(MAJOR, E_INVALID_STATE, NO_MSG); + } #ifdef FM_UCODE_NOT_RESET_ERRATA_BUGZILLA6173 } #endif /* FM_UCODE_NOT_RESET_ERRATA_BUGZILLA6173 */ @@ -4825,6 +4831,86 @@ uint32_t FM_GetCounter(t_Handle h_Fm, e_FmCounters counter) return fman_get_counter(&fman_rg, fsl_counter); } +#ifdef __aarch64__ +void FM_ErrorDiag(t_Handle h_Fm, uint8_t hwPortId) +{ + t_Fm *p = (t_Fm *)h_Fm; + uint32_t ps, epi, cld, npi, gs, ts, eie, dmsr, ievr; + uint32_t dmtah, dmtal; + + /* FPM: port stall status + error pending */ + ps = ioread32be(&p->p_FmFpmRegs->fmfp_ps[hwPortId]); + epi = ioread32be(&p->p_FmFpmRegs->fm_epi); + cld = ioread32be(&p->p_FmFpmRegs->fm_cld); + npi = ioread32be(&p->p_FmFpmRegs->fm_npi); + + /* QMI: global status + task status + error */ + gs = ioread32be(&p->p_FmQmiRegs->fmqm_gs); + ts = ioread32be(&p->p_FmQmiRegs->fmqm_ts); + eie = ioread32be(&p->p_FmQmiRegs->fmqm_eie); + + /* DMA: status + last transfer address */ + dmsr = ioread32be(&p->p_FmDmaRegs->fmdmsr); + dmtah = ioread32be(&p->p_FmDmaRegs->fmdmtah); + dmtal = ioread32be(&p->p_FmDmaRegs->fmdmtal); + + /* BMI: interrupt event */ + ievr = ioread32be(&p->p_FmBmiRegs->fmbm_ievr); + + printf("fman: FPM port[%u] ps=0x%08x %s\n", + hwPortId, ps, (ps & 0x00800000) ? "STALLED" : "ok"); + printf("fman: FPM epi=0x%08x npi=0x%08x cld=0x%08x\n", epi, npi, cld); + printf("fman: QMI gs=0x%08x ts=0x%08x eie=0x%08x\n", gs, ts, eie); + printf("fman: DMA sr=0x%08x addr=0x%08x%08x\n", dmsr, dmtah, dmtal); + printf("fman: BMI ievr=0x%08x\n", ievr); +} + +void FM_TaskStatusDiag(t_Handle h_Fm, uint8_t hwPortId) +{ + t_Fm *p = (t_Fm *)h_Fm; + uint32_t pp, ts; + int t, port_tasks = 0, total_active = 0; + uint8_t port_count[256]; + int i, dumped = 0; + + if (hwPortId == 0 || hwPortId > 63) + return; + + memset(port_count, 0, sizeof(port_count)); + + /* Read configured tasks from BMI Port Parameters */ + pp = ioread32be(&p->p_FmBmiRegs->fmbm_pp[hwPortId - 1]); + printf("fman: BMI pp[%u]=0x%08x (tasks=%u, extra=%u)\n", + hwPortId, pp, + ((pp >> 24) & 0x3f) + 1, + (pp >> 16) & 0xf); + + /* Scan FPM task status registers — dump first 8 non-zero raw values */ + for (t = 0; t < 128; t++) { + ts = ioread32be(&p->p_FmFpmRegs->fmfp_ts[t]); + if (ts != 0) { + uint8_t owner = (ts >> 24) & 0xff; + total_active++; + port_count[owner]++; + if (owner == hwPortId) + port_tasks++; + if (dumped < 8) { + printf("fman: ts[%d]=0x%08x (owner=%u)\n", t, ts, owner); + dumped++; + } + } + } + + /* Print per-port task breakdown for all ports with active tasks */ + printf("fman: task breakdown:"); + for (i = 0; i < 256; i++) { + if (port_count[i] > 0) + printf(" port%d=%d", i, port_count[i]); + } + printf(" (total=%d, port%u=%d)\n", total_active, hwPortId, port_tasks); +} +#endif + t_Error FM_ModifyCounter(t_Handle h_Fm, e_FmCounters counter, uint32_t val) { t_Fm *p_Fm = (t_Fm*)h_Fm; @@ -5191,7 +5277,7 @@ t_Handle FmGetPcd(t_Handle h_Fm) extern void *g_MemacRegs; void fm_clk_down(void); uint32_t fman_memac_get_event(void *regs, uint32_t ev_mask); -void FM_ChangeClock(t_Handle h_Fm, int hardwarePortId) +static void FM_ChangeClock(t_Handle h_Fm, int hardwarePortId) { int macId; uint32_t event, rcr; diff --git a/sys/contrib/ncsw/Peripherals/FM/inc/fm_common.h b/sys/contrib/ncsw/Peripherals/FM/inc/fm_common.h index 81a25e976b62..354e72773897 100644 --- a/sys/contrib/ncsw/Peripherals/FM/inc/fm_common.h +++ b/sys/contrib/ncsw/Peripherals/FM/inc/fm_common.h @@ -52,7 +52,8 @@ #define IP_OFFLOAD_PACKAGE_NUMBER 106 #define CAPWAP_OFFLOAD_PACKAGE_NUMBER 108 -#define IS_OFFLOAD_PACKAGE(num) ((num == IP_OFFLOAD_PACKAGE_NUMBER) || (num == CAPWAP_OFFLOAD_PACKAGE_NUMBER)) +#define ASK_UCODE_PACKAGE_NUMBER 209 +#define IS_OFFLOAD_PACKAGE(num) ((num == IP_OFFLOAD_PACKAGE_NUMBER) || (num == CAPWAP_OFFLOAD_PACKAGE_NUMBER) || (num >= ASK_UCODE_PACKAGE_NUMBER)) @@ -189,7 +190,13 @@ typedef _Packed struct t_FmPcdCtrlParamsPage { volatile uint32_t discardMask; volatile uint8_t reserved3[4]; volatile uint32_t postBmiFetchNia; +#if (DPAA_VERSION >= 11) + volatile uint32_t internalFEBufferManagementIndexAddr; + volatile uint32_t internalFEBufferDepletionCounter; + volatile uint8_t reserved4[164]; +#else volatile uint8_t reserved4[172]; +#endif } _PackedType t_FmPcdCtrlParamsPage; diff --git a/sys/contrib/ncsw/Peripherals/QM/qm.c b/sys/contrib/ncsw/Peripherals/QM/qm.c index ed3049f0a318..dbe25ccb3642 100644 --- a/sys/contrib/ncsw/Peripherals/QM/qm.c +++ b/sys/contrib/ncsw/Peripherals/QM/qm.c @@ -295,9 +295,9 @@ static uint32_t ReserveFqids(t_Qm *p_Qm, uint32_t size, uint32_t alignment, bool static void FreeInitResources(t_Qm *p_Qm) { - if (p_Qm->p_FqdBase) + if (p_Qm->p_FqdBase && !p_Qm->fqdExternal) XX_FreeSmart(p_Qm->p_FqdBase); - if (p_Qm->p_PfdrBase) + if (p_Qm->p_PfdrBase && !p_Qm->pfdrExternal) XX_FreeSmart(p_Qm->p_PfdrBase); if (p_Qm->h_Session) XX_IpcFreeSession(p_Qm->h_Session); @@ -705,6 +705,10 @@ t_Handle QM_Config(t_QmParam *p_QmParam) p_Qm->p_QmDriverParams->pfdrThreshold = DEFAULT_pfdrThreshold; p_Qm->p_QmDriverParams->sfdrThreshold = DEFAULT_sfdrThreshold; p_Qm->p_QmDriverParams->pfdrBaseConstant = DEFAULT_pfdrBaseConstant; + p_Qm->p_QmDriverParams->p_FqdBaseExt = p_QmParam->p_FqdBase; + p_Qm->p_QmDriverParams->fqdSizeExt = p_QmParam->fqdSize; + p_Qm->p_QmDriverParams->p_PfdrBaseExt = p_QmParam->p_PfdrBase; + p_Qm->p_QmDriverParams->pfdrSizeExt = p_QmParam->pfdrSize; for(i= 0;ip_QmDriverParams->dcPortalsParams[i].sendToSw = (bool)((i < e_DPAA_DCPORTAL2) ? FALSE : TRUE); @@ -774,19 +778,32 @@ t_Error QM_Init(t_Handle h_Qm) WRITE_UINT32(p_Qm->p_QmRegs->liodnr, (uint16_t)p_QmDriverParams->liodn); /* FQD memory */ - dsSize = (uint32_t)(p_QmDriverParams->totalNumOfFqids * FQD_ENTRY_SIZE); - LOG2(dsSize, exp); - if (!POWER_OF_2(dsSize)) (exp++); - dsSize = (uint32_t)(1 << exp); - if (dsSize < (4*KILOBYTE)) + if (p_QmDriverParams->p_FqdBaseExt) { - dsSize = (4*KILOBYTE); + /* Use pre-allocated FQD memory from caller (e.g. DT reserved-memory) */ + p_Qm->p_FqdBase = p_QmDriverParams->p_FqdBaseExt; + p_Qm->fqdExternal = TRUE; + dsSize = p_QmDriverParams->fqdSizeExt; LOG2(dsSize, exp); + if (!POWER_OF_2(dsSize)) (exp++); + dsSize = (uint32_t)(1 << exp); } - p_Qm->p_FqdBase = XX_MallocSmart(dsSize, (int)p_QmDriverParams->fqdMemPartitionId, dsSize); - if (!p_Qm->p_FqdBase) + else { - RETURN_ERROR(MAJOR, E_NO_MEMORY, ("FQD obj!!!")); + dsSize = (uint32_t)(p_QmDriverParams->totalNumOfFqids * FQD_ENTRY_SIZE); + LOG2(dsSize, exp); + if (!POWER_OF_2(dsSize)) (exp++); + dsSize = (uint32_t)(1 << exp); + if (dsSize < (4*KILOBYTE)) + { + dsSize = (4*KILOBYTE); + LOG2(dsSize, exp); + } + p_Qm->p_FqdBase = XX_MallocSmart(dsSize, (int)p_QmDriverParams->fqdMemPartitionId, dsSize); + if (!p_Qm->p_FqdBase) + { + RETURN_ERROR(MAJOR, E_NO_MEMORY, ("FQD obj!!!")); + } } memset(p_Qm->p_FqdBase, 0, dsSize); mb(); @@ -799,29 +816,67 @@ t_Error QM_Init(t_Handle h_Qm) WRITE_UINT32(p_Qm->p_QmRegs->fqd_bar, (uint32_t)phyAddr); WRITE_UINT32(p_Qm->p_QmRegs->fqd_ar, AR_ENABLE | (exp - 1)); + /* Readback and verify FQD BAR programming */ + { + uint32_t rb_bare = GET_UINT32(p_Qm->p_QmRegs->fqd_bare); + uint32_t rb_bar = GET_UINT32(p_Qm->p_QmRegs->fqd_bar); + uint32_t rb_ar = GET_UINT32(p_Qm->p_QmRegs->fqd_ar); + XX_Print("QM FQD: va=%p phys=0x%08x%08x bare=0x%08x bar=0x%08x ar=0x%08x (exp=%d, size=0x%x)\n", + p_Qm->p_FqdBase, + (uint32_t)(phyAddr >> 32), (uint32_t)phyAddr, + rb_bare, rb_bar, rb_ar, exp, dsSize); + } + /* PFDR memory */ - dsSize = (uint32_t)(p_QmDriverParams->rtFramesDepth * (PFDR_ENTRY_SIZE/3)); - LOG2(dsSize, exp); - if (!POWER_OF_2(dsSize)) (exp++); - dsSize = (uint32_t)(1 << exp); - if (dsSize < (4*KILOBYTE)) + if (p_QmDriverParams->p_PfdrBaseExt) { - dsSize = (4*KILOBYTE); + /* Use pre-allocated PFDR memory from caller */ + p_Qm->p_PfdrBase = p_QmDriverParams->p_PfdrBaseExt; + p_Qm->pfdrExternal = TRUE; + dsSize = p_QmDriverParams->pfdrSizeExt; LOG2(dsSize, exp); + if (!POWER_OF_2(dsSize)) (exp++); + dsSize = (uint32_t)(1 << exp); + } + else + { + dsSize = (uint32_t)(p_QmDriverParams->rtFramesDepth * (PFDR_ENTRY_SIZE/3)); + LOG2(dsSize, exp); + if (!POWER_OF_2(dsSize)) (exp++); + dsSize = (uint32_t)(1 << exp); + if (dsSize < (4*KILOBYTE)) + { + dsSize = (4*KILOBYTE); + LOG2(dsSize, exp); + } + p_Qm->p_PfdrBase = XX_MallocSmart(dsSize, (int)p_QmDriverParams->pfdrMemPartitionId, dsSize); + if (!p_Qm->p_PfdrBase) + RETURN_ERROR(MAJOR, E_NO_MEMORY, ("PFDR obj!!!")); } - - p_Qm->p_PfdrBase = XX_MallocSmart(dsSize, (int)p_QmDriverParams->pfdrMemPartitionId, dsSize); - if (!p_Qm->p_PfdrBase) - RETURN_ERROR(MAJOR, E_NO_MEMORY, ("PFDR obj!!!")); phyAddr = XX_VirtToPhys(p_Qm->p_PfdrBase); WRITE_UINT32(p_Qm->p_QmRegs->pfdr_bare, ((uint32_t)(phyAddr >> 32) & 0x000000ff)); WRITE_UINT32(p_Qm->p_QmRegs->pfdr_bar, (uint32_t)phyAddr); WRITE_UINT32(p_Qm->p_QmRegs->pfdr_ar, AR_ENABLE | (exp - 1)); + /* Readback and verify PFDR BAR programming */ + { + uint32_t rb_bare = GET_UINT32(p_Qm->p_QmRegs->pfdr_bare); + uint32_t rb_bar = GET_UINT32(p_Qm->p_QmRegs->pfdr_bar); + uint32_t rb_ar = GET_UINT32(p_Qm->p_QmRegs->pfdr_ar); + XX_Print("QM PFDR: va=%p phys=0x%08x%08x bare=0x%08x bar=0x%08x ar=0x%08x (exp=%d, size=0x%x)\n", + p_Qm->p_PfdrBase, + (uint32_t)(phyAddr >> 32), (uint32_t)phyAddr, + rb_bare, rb_bar, rb_ar, exp, dsSize); + XX_Print("QM PFDR init: pfdr_start=8, num=%d\n", dsSize / 64 - 8); + } + if (QmInitPfdr(p_Qm, 8, dsSize / 64 - 8) != E_OK) RETURN_ERROR(MAJOR, E_INVALID_STATE, ("PFDR init failed!!!")); + XX_Print("QM PFDR init: SUCCESS, pfdr_fpc=%d\n", + GET_UINT32(p_Qm->p_QmRegs->pfdr_fpc)); + /* thresholds */ WRITE_UINT32(p_Qm->p_QmRegs->pfdr_fp_lwit, (p_Qm->p_QmDriverParams->pfdrThreshold & 0xffffff)); WRITE_UINT32(p_Qm->p_QmRegs->pfdr_cfg, p_Qm->p_QmDriverParams->pfdrBaseConstant); @@ -1037,6 +1092,27 @@ uint32_t QM_GetCounter(t_Handle h_Qm, e_QmCounters counter) return 0; } +#ifdef __aarch64__ +void QM_ErrorDiag(t_Handle h_Qm, uint32_t *err_isr, uint32_t *ecsr, + uint32_t *ecir, uint32_t *pfdr_fpc, uint32_t *sfdr_in_use, + uint32_t *idle_stat, uint32_t *dcp0_cfg, uint32_t *dcp0_dlm_avg, + uint32_t *dcp_dd_ihrsr, uint32_t *dcp_dd_hasr) +{ + t_Qm *p_Qm = (t_Qm *)h_Qm; + + *err_isr = GET_UINT32(p_Qm->p_QmRegs->err_isr); + *ecsr = GET_UINT32(p_Qm->p_QmRegs->ecsr); + *ecir = GET_UINT32(p_Qm->p_QmRegs->ecir); + *pfdr_fpc = GET_UINT32(p_Qm->p_QmRegs->pfdr_fpc); + *sfdr_in_use = GET_UINT32(p_Qm->p_QmRegs->sfdr_in_use); + *idle_stat = GET_UINT32(p_Qm->p_QmRegs->idle_stat); + *dcp0_cfg = GET_UINT32(p_Qm->p_QmRegs->dcpConfRegs[0].cfg); + *dcp0_dlm_avg = GET_UINT32(p_Qm->p_QmRegs->dcpConfRegs[0].dlm_avg); + *dcp_dd_ihrsr = GET_UINT32(p_Qm->p_QmRegs->dcp_dd_ihrsr); + *dcp_dd_hasr = GET_UINT32(p_Qm->p_QmRegs->dcp_dd_hasr); +} +#endif + void QM_ErrorIsr(t_Handle h_Qm) { t_Qm *p_Qm = (t_Qm *)h_Qm; diff --git a/sys/contrib/ncsw/Peripherals/QM/qm.h b/sys/contrib/ncsw/Peripherals/QM/qm.h index a004dc55a67d..a18ada7c719d 100644 --- a/sys/contrib/ncsw/Peripherals/QM/qm.h +++ b/sys/contrib/ncsw/Peripherals/QM/qm.h @@ -373,7 +373,11 @@ typedef _Packed struct #define QM_CGR_TARGET_DCP(portlaId) (QM_CGR_TARG_FIRST_DCPORTAL >> portlaId) -#define QM_DCP_CFG_ED 0x00000100 +#ifdef __aarch64__ +#define QM_DCP_CFG_ED 0x00001000 /* QMan v3 (LS1046A) */ +#else +#define QM_DCP_CFG_ED 0x00000100 /* QMan v1/v2 (P5020) */ +#endif /* #define CGR_VALID 0x80 #define CGR_VERB_INIT 0x50 @@ -467,6 +471,10 @@ typedef struct { uint32_t pfdrBaseConstant; uint16_t liodn; t_QmDcPortalParams dcPortalsParams[DPAA_MAX_NUM_OF_DC_PORTALS]; + void *p_FqdBaseExt; /* Pre-allocated FQD, or NULL */ + uint32_t fqdSizeExt; + void *p_PfdrBaseExt; /* Pre-allocated PFDR, or NULL */ + uint32_t pfdrSizeExt; } t_QmDriverParams; typedef struct { @@ -479,6 +487,8 @@ typedef struct { t_QmRegs *p_QmRegs; uint32_t *p_FqdBase; uint32_t *p_PfdrBase; + bool fqdExternal; /* FQD memory externally allocated */ + bool pfdrExternal; /* PFDR memory externally allocated */ uint32_t exceptions; t_QmExceptionsCallback *f_Exception; t_Handle h_App; diff --git a/sys/contrib/ncsw/Peripherals/QM/qm_portal_fqr.c b/sys/contrib/ncsw/Peripherals/QM/qm_portal_fqr.c index 31313d37f97c..33d630fafa6b 100644 --- a/sys/contrib/ncsw/Peripherals/QM/qm_portal_fqr.c +++ b/sys/contrib/ncsw/Peripherals/QM/qm_portal_fqr.c @@ -1,6 +1,6 @@ /****************************************************************************** - © 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc. + � 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc. All rights reserved. This is proprietary source code of Freescale Semiconductor Inc., @@ -64,22 +64,69 @@ * call. Take advantage of this fact to shove a 64-bit kernel pointer into a * 32-bit context integer, and back. * - * XXX: This depends on the fact that VM_MAX_KERNEL_ADDRESS is less than 38-bit - * count from VM_MIN_KERNEL_ADDRESS. If this ever changes, this needs to be - * updated. + * On ARM64, encode the physical address shifted by 6 (64-byte alignment). + * XX_MallocSmart uses malloc() which returns DMAP addresses — virtual + * address encoding is not feasible because the DMAP-to-KVA offset exceeds + * 32 bits. Physical addresses on LS1046A are ≤10GB, so shifted by 6 they + * fit trivially in 32 bits (max ~160M). Reconstruction uses XX_PhysToVirt + * which returns the DMAP virtual address. + * + * On PowerPC, shift by 3 (8-byte alignment) relative to VM_MIN_KERNEL_ADDRESS. */ +#if defined(__aarch64__) +#define QMAN_CTX_SHIFT 6 +static inline uint32_t +aligned_int_from_ptr(const void *p) +{ + physAddress_t pa; + uint32_t result; + + pa = XX_VirtToPhys((void *)(uintptr_t)p); + KASSERT((pa & ((1 << QMAN_CTX_SHIFT) - 1)) == 0, + ("Pointer %p (pa %#lx) is not %d-byte aligned!\n", + p, (unsigned long)pa, 1 << QMAN_CTX_SHIFT)); + + result = (uint32_t)(pa >> QMAN_CTX_SHIFT); + KASSERT((physAddress_t)result == (pa >> QMAN_CTX_SHIFT), + ("%s: %p (pa %#lx) overflows 32-bit context\n", + __func__, p, (unsigned long)pa)); + + return (result); +} + +static inline void * +ptr_from_aligned_int(uint32_t ctx) +{ + physAddress_t pa; + + pa = (physAddress_t)ctx << QMAN_CTX_SHIFT; + + return (XX_PhysToVirt(pa)); +} +#else CTASSERT((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) < (1ULL << 35)); +#define QMAN_CTX_SHIFT 3 +#define QMAN_CTX_BASE VM_MIN_KERNEL_ADDRESS static inline uint32_t aligned_int_from_ptr(const void *p) { uintptr_t ctx; + uint32_t result; ctx = (uintptr_t)p; - KASSERT(ctx >= VM_MIN_KERNEL_ADDRESS, ("%p is too low!\n", p)); - ctx -= VM_MIN_KERNEL_ADDRESS; - KASSERT((ctx & 0x07) == 0, ("Pointer %p is not 8-byte aligned!\n", p)); - - return (ctx >> 3); + KASSERT(ctx >= QMAN_CTX_BASE, + ("%s: %p is below base %#lx\n", __func__, p, + (unsigned long)QMAN_CTX_BASE)); + ctx -= QMAN_CTX_BASE; + KASSERT((ctx & ((1 << QMAN_CTX_SHIFT) - 1)) == 0, + ("Pointer %p is not %d-byte aligned!\n", p, 1 << QMAN_CTX_SHIFT)); + + result = (ctx >> QMAN_CTX_SHIFT); + KASSERT((uintptr_t)result == (ctx >> QMAN_CTX_SHIFT), + ("%s: %p overflows 32-bit context (shifted %#lx)\n", + __func__, p, (unsigned long)(ctx >> QMAN_CTX_SHIFT))); + + return (result); } static inline void * @@ -88,10 +135,59 @@ ptr_from_aligned_int(uint32_t ctx) uintptr_t p; p = ctx; - p = VM_MIN_KERNEL_ADDRESS + (p << 3); + p = QMAN_CTX_BASE + (p << QMAN_CTX_SHIFT); return ((void *)p); } +#endif + +/* + * Copy a frame descriptor from CE portal memory (big-endian) to a native + * t_DpaaFD struct with proper byte-swapping. On big-endian (PowerPC), + * GET_UINT32 is a plain load so this is equivalent to a raw copy. + * + * On LE (ARM64), t_DpaaFD swaps the word order of the first 8 bytes: + * HW (BE): [liodn,bpid,elion,addrh][addrl_BE] + * t_DpaaFD LE: [addrl_LE][addrh,elion,bpid,liodn] + * So we swap word positions AND byte-swap each word (equivalent to bswap64). + */ +static inline void +qm_fd_read(t_DpaaFD *dst, const volatile struct qm_fd *src) +{ + const volatile uint32_t *s = (const volatile uint32_t *)src; + uint32_t *d = (uint32_t *)dst; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + d[0] = s[0]; + d[1] = s[1]; +#else + d[0] = GET_UINT32(s[1]); + d[1] = GET_UINT32(s[0]); +#endif + d[2] = GET_UINT32(s[2]); + d[3] = GET_UINT32(s[3]); +} + +/* + * Copy a native t_DpaaFD to CE portal memory (big-endian) with proper + * byte-swapping. Reverse of qm_fd_read(). + */ +static inline void +qm_fd_write(volatile struct qm_fd *dst, const t_DpaaFD *src) +{ + volatile uint32_t *d = (volatile uint32_t *)dst; + const uint32_t *s = (const uint32_t *)src; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + d[0] = s[0]; + d[1] = s[1]; +#else + WRITE_UINT32(d[0], s[1]); + WRITE_UINT32(d[1], s[0]); +#endif + WRITE_UINT32(d[2], s[2]); + WRITE_UINT32(d[3], s[3]); +} static t_Error qman_volatile_dequeue(t_QmPortal *p_QmPortal, struct qman_fq *p_Fq, @@ -152,7 +248,7 @@ static t_Error qman_create_fq(t_QmPortal *p_QmPortal, /* Everything else is RECOVER support */ NCSW_PLOCK(p_QmPortal); p_Mcc = qm_mc_start(p_QmPortal->p_LowQmPortal); - p_Mcc->queryfq.fqid = fqid; + WRITE_UINT32(p_Mcc->queryfq.fqid, fqid); qm_mc_commit(p_QmPortal->p_LowQmPortal, QM_MCC_VERB_QUERYFQ); while (!(p_Mcr = qm_mc_result(p_QmPortal->p_LowQmPortal))) ; ASSERT_COND((p_Mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); @@ -162,7 +258,7 @@ static t_Error qman_create_fq(t_QmPortal *p_QmPortal, } fqd = p_Mcr->queryfq.fqd; p_Mcc = qm_mc_start(p_QmPortal->p_LowQmPortal); - p_Mcc->queryfq_np.fqid = fqid; + WRITE_UINT32(p_Mcc->queryfq_np.fqid, fqid); qm_mc_commit(p_QmPortal->p_LowQmPortal, QM_MCC_VERB_QUERYFQ_NP); while (!(p_Mcr = qm_mc_result(p_QmPortal->p_LowQmPortal))) ; ASSERT_COND((p_Mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); @@ -245,7 +341,56 @@ static t_Error qman_init_fq(t_QmPortal *p_QmPortal, return ERROR_CODE(E_BUSY); } p_Mcc = qm_mc_start(p_QmPortal->p_LowQmPortal); +#ifdef __aarch64__ + /* + * Portal CE memory is big-endian on QorIQ ARM64 (LS1046A). + * Mem2IOCpy32 doesn't handle LE→BE conversion correctly for packed + * structs whose fields span 32-bit word boundaries — the byte-by-byte + * prefix and the unaligned shift-merge were designed for big-endian + * PowerPC only. Write each field individually using WRITE_UINTxx + * (out{8,16,32}rb) which byte-swap LE values to BE portal bytes. + * + * Bitfield-containing sub-fields (dest_wq, td_thresh) must be manually + * reconstructed in BE bit layout because GCC allocates bitfields from + * LSB on LE vs MSB on BE. + */ + WRITE_UINT8(p_Mcc->initfq.reserved1, 0); + WRITE_UINT16(p_Mcc->initfq.we_mask, p_Opts->we_mask); + WRITE_UINT32(p_Mcc->initfq.fqid, p_Opts->fqid); + WRITE_UINT16(p_Mcc->initfq.count, p_Opts->count); + /* FQD sub-struct */ + WRITE_UINT8(p_Mcc->initfq.fqd.orpc, p_Opts->fqd.orpc); + WRITE_UINT8(p_Mcc->initfq.fqd.cgid, p_Opts->fqd.cgid); + WRITE_UINT16(p_Mcc->initfq.fqd.fq_ctrl, p_Opts->fqd.fq_ctrl); + { + /* dest_wq has bitfields channel:13 wq:3. + * BE bit layout: channel(bits15-3) wq(bits2-0). */ + uint16_t dest_wq_be = ((uint16_t)(p_Opts->fqd.dest.channel & 0x1FFF) << 3) | + (p_Opts->fqd.dest.wq & 0x7); + WRITE_UINT16(p_Mcc->initfq.fqd.dest_wq, dest_wq_be); + } + { + /* reserved2:1 + ics_cred:15 are bare bitfields with no union accessor. + * Address the 16-bit word via the preceding dest_wq union. */ + volatile uint16_t *p_ics = (volatile uint16_t *)&p_Mcc->initfq.fqd.dest_wq + 1; + /* BE bit layout: reserved2(bit15) ics_cred(bits14-0). reserved2=0. */ + uint16_t ics_be = (uint16_t)(p_Opts->fqd.ics_cred & 0x7FFF); + WRITE_UINT16(*p_ics, ics_be); + } + { + /* td_thresh has bitfields reserved1:3 mant:8 exp:5. + * BE bit layout: reserved(bits15-13) mant(bits12-5) exp(bits4-0). */ + uint16_t td_be = ((uint16_t)(p_Opts->fqd.td.mant & 0xFF) << 5) | + (p_Opts->fqd.td.exp & 0x1F); + WRITE_UINT16(p_Mcc->initfq.fqd.td_thresh, td_be); + } + WRITE_UINT32(p_Mcc->initfq.fqd.context_b, p_Opts->fqd.context_b); + WRITE_UINT32(p_Mcc->initfq.fqd.context_a.hi, p_Opts->fqd.context_a.hi); + WRITE_UINT32(p_Mcc->initfq.fqd.context_a.lo, p_Opts->fqd.context_a.lo); + /* reserved3[32] already zeroed by dcbz_64 in qm_mc_start */ +#else Mem2IOCpy32((void*)&p_Mcc->initfq, p_Opts, sizeof(struct qm_mcc_initfq)); +#endif qm_mc_commit(p_QmPortal->p_LowQmPortal, myverb); while (!(p_Mcr = qm_mc_result(p_QmPortal->p_LowQmPortal))) ; ASSERT_COND((p_Mcr->verb & QM_MCR_VERB_MASK) == myverb); @@ -253,17 +398,19 @@ static t_Error qman_init_fq(t_QmPortal *p_QmPortal, if (res != QM_MCR_RESULT_OK) { FQUNLOCK(p_Fq); PUNLOCK(p_QmPortal); - RETURN_ERROR(MINOR, E_INVALID_STATE,("INITFQ failed: %s", mcr_result_str(res))); + RETURN_ERROR(MINOR, E_INVALID_STATE,("INITFQ failed: fqid=%d %s", + p_Fq->fqid, mcr_result_str(res))); } - if (p_Mcc->initfq.we_mask & QM_INITFQ_WE_FQCTRL) { - if (p_Mcc->initfq.fqd.fq_ctrl & QM_FQCTRL_CGE) + /* Read flags back from the source struct, not portal CE (which is BE) */ + if (p_Opts->we_mask & QM_INITFQ_WE_FQCTRL) { + if (p_Opts->fqd.fq_ctrl & QM_FQCTRL_CGE) p_Fq->flags |= QMAN_FQ_STATE_CGR_EN; else p_Fq->flags &= ~QMAN_FQ_STATE_CGR_EN; } - if (p_Mcc->initfq.we_mask & QM_INITFQ_WE_CGID) - p_Fq->cgr_groupid = p_Mcc->initfq.fqd.cgid; + if (p_Opts->we_mask & QM_INITFQ_WE_CGID) + p_Fq->cgr_groupid = p_Opts->fqd.cgid; p_Fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? qman_fq_state_sched : qman_fq_state_parked; FQUNLOCK(p_Fq); @@ -296,9 +443,9 @@ static t_Error qman_retire_fq(t_QmPortal *p_QmPortal, goto out; } p_Mcc = qm_mc_start(p_QmPortal->p_LowQmPortal); - p_Mcc->alterfq.fqid = p_Fq->fqid; + WRITE_UINT32(p_Mcc->alterfq.fqid, p_Fq->fqid); if (drain) - p_Mcc->alterfq.context_b = aligned_int_from_ptr(p_Fq); + WRITE_UINT32(p_Mcc->alterfq.context_b, aligned_int_from_ptr(p_Fq)); qm_mc_commit(p_QmPortal->p_LowQmPortal, (uint8_t)((drain)?QM_MCC_VERB_ALTER_RETIRE_CTXB:QM_MCC_VERB_ALTER_RETIRE)); while (!(p_Mcr = qm_mc_result(p_QmPortal->p_LowQmPortal))) ; @@ -347,7 +494,7 @@ static t_Error qman_oos_fq(t_QmPortal *p_QmPortal, struct qman_fq *p_Fq) return ERROR_CODE(E_BUSY); } p_Mcc = qm_mc_start(p_QmPortal->p_LowQmPortal); - p_Mcc->alterfq.fqid = p_Fq->fqid; + WRITE_UINT32(p_Mcc->alterfq.fqid, p_Fq->fqid); qm_mc_commit(p_QmPortal->p_LowQmPortal, QM_MCC_VERB_ALTER_OOS); while (!(p_Mcr = qm_mc_result(p_QmPortal->p_LowQmPortal))) ; ASSERT_COND((p_Mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); @@ -383,7 +530,7 @@ static t_Error qman_schedule_fq(t_QmPortal *p_QmPortal, struct qman_fq *p_Fq) return ERROR_CODE(E_BUSY); } p_Mcc = qm_mc_start(p_QmPortal->p_LowQmPortal); - p_Mcc->alterfq.fqid = p_Fq->fqid; + WRITE_UINT32(p_Mcc->alterfq.fqid, p_Fq->fqid); qm_mc_commit(p_QmPortal->p_LowQmPortal, QM_MCC_VERB_ALTER_SCHED); while (!(p_Mcr = qm_mc_result(p_QmPortal->p_LowQmPortal))) ; ASSERT_COND((p_Mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); @@ -607,14 +754,30 @@ static uint32_t LoopMessageRing(t_QmPortal *p_QmPortal, uint32_t is) qmPortalMrPvbUpdate(p_QmPortal->p_LowQmPortal); p_Msg = qm_mr_current(p_QmPortal->p_LowQmPortal); if (p_Msg) { - struct qman_fq *p_FqFqs = ptr_from_aligned_int(p_Msg->fq.contextB); - struct qman_fq *p_FqErn = ptr_from_aligned_int(p_Msg->ern.tag); + struct qman_fq *p_FqFqs = ptr_from_aligned_int(GET_UINT32(p_Msg->fq.contextB)); + struct qman_fq *p_FqErn = ptr_from_aligned_int(GET_UINT32(p_Msg->ern.tag)); uint8_t verb =(uint8_t)(p_Msg->verb & QM_MR_VERB_TYPE_MASK); t_QmRejectedFrameInfo rejectedFrameInfo; + t_DpaaFD mr_fd; memset(&rejectedFrameInfo, 0, sizeof(t_QmRejectedFrameInfo)); if (!(verb & QM_MR_VERB_DC_ERN)) { + qm_fd_read(&mr_fd, &p_Msg->ern.fd); + { + static int ern_print_count; + if (ern_print_count++ < 20) + XX_Print("QM ERN: verb=0x%02x rc=0x%02x fqid=%u " + "fd_addr=0x%08x%08x fd_status=0x%x\n", + verb, p_Msg->ern.rc, + p_FqErn ? p_FqErn->fqid : 0, + DPAA_FD_GET_ADDRH(&mr_fd), + DPAA_FD_GET_ADDRL(&mr_fd), + DPAA_FD_GET_STATUS(&mr_fd)); + } + if (DPAA_FD_GET_PHYS_ADDR(&mr_fd) == 0) + XX_Print("LoopMessageRing: ERN FD.addr=0 verb=0x%02x rc=0x%02x\n", + verb, p_Msg->ern.rc); switch(p_Msg->ern.rc) { case(QM_MR_RC_CGR_TAILDROP): @@ -635,11 +798,17 @@ static uint32_t LoopMessageRing(t_QmPortal *p_QmPortal, uint32_t is) REPORT_ERROR(MINOR, E_NOT_SUPPORTED, ("Unknown rejection code")); } if (!p_FqErn) - p_QmPortal->p_NullCB->ern(p_QmPortal->h_App, NULL, p_QmPortal, 0, (t_DpaaFD*)&p_Msg->ern.fd, &rejectedFrameInfo); + p_QmPortal->p_NullCB->ern(p_QmPortal->h_App, NULL, p_QmPortal, 0, &mr_fd, &rejectedFrameInfo); else - p_FqErn->cb.ern(p_FqErn->h_App, p_FqErn->h_QmFqr, p_QmPortal, p_FqErn->fqidOffset, (t_DpaaFD*)&p_Msg->ern.fd, &rejectedFrameInfo); + p_FqErn->cb.ern(p_FqErn->h_App, p_FqErn->h_QmFqr, p_QmPortal, p_FqErn->fqidOffset, &mr_fd, &rejectedFrameInfo); } else if (verb == QM_MR_VERB_DC_ERN) { + { + static int dc_ern_count; + if (dc_ern_count++ < 20) + XX_Print("QM DC_ERN: fqid=%u\n", + p_FqErn ? p_FqErn->fqid : 0); + } if (!p_FqErn) p_QmPortal->p_NullCB->dc_ern(NULL, p_QmPortal, NULL, p_Msg); else @@ -676,6 +845,7 @@ static void LoopDequeueRing(t_Handle h_QmPortal) e_RxStoreResponse tmpRes; t_QmPortal *p_QmPortal = (t_QmPortal *)h_QmPortal; int prefetch = !(p_QmPortal->options & QMAN_PORTAL_FLAG_RSTASH); + t_DpaaFD fd; while (res != qman_cb_dqrr_pause) { @@ -685,12 +855,13 @@ static void LoopDequeueRing(t_Handle h_QmPortal) p_Dq = qm_dqrr_current(p_QmPortal->p_LowQmPortal); if (!p_Dq) break; - p_Fq = ptr_from_aligned_int(p_Dq->contextB); + p_Fq = ptr_from_aligned_int(GET_UINT32(p_Dq->contextB)); + qm_fd_read(&fd, &p_Dq->fd); if (p_Dq->stat & QM_DQRR_STAT_UNSCHEDULED) { /* We only set QMAN_FQ_STATE_NE when retiring, so we only need * to check for clearing it when doing volatile dequeues. It's * one less thing to check in the critical path (SDQCR). */ - tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, (t_DpaaFD*)&p_Dq->fd); + tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; /* Check for VDQCR completion */ @@ -706,14 +877,14 @@ static void LoopDequeueRing(t_Handle h_QmPortal) { /* Interpret 'dq' from the owner's perspective. */ /* use portal default handlers */ - ASSERT_COND(p_Dq->fqid); + ASSERT_COND(GET_UINT32(p_Dq->fqid)); if (p_Fq) { tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, - (t_DpaaFD*)&p_Dq->fd); + &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; else if (p_Fq->state == qman_fq_state_waiting_parked) @@ -724,8 +895,8 @@ static void LoopDequeueRing(t_Handle h_QmPortal) tmpRes = p_QmPortal->p_NullCB->dqrr(p_QmPortal->h_App, NULL, p_QmPortal, - p_Dq->fqid, - (t_DpaaFD*)&p_Dq->fd); + GET_UINT32(p_Dq->fqid), + &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; } @@ -760,6 +931,7 @@ static void LoopDequeueRingDcaOptimized(t_Handle h_QmPortal) enum qman_cb_dqrr_result res = qman_cb_dqrr_consume; e_RxStoreResponse tmpRes; t_QmPortal *p_QmPortal = (t_QmPortal *)h_QmPortal; + t_DpaaFD fd; while (res != qman_cb_dqrr_pause) { @@ -767,12 +939,13 @@ static void LoopDequeueRingDcaOptimized(t_Handle h_QmPortal) p_Dq = qm_dqrr_current(p_QmPortal->p_LowQmPortal); if (!p_Dq) break; - p_Fq = ptr_from_aligned_int(p_Dq->contextB); + p_Fq = ptr_from_aligned_int(GET_UINT32(p_Dq->contextB)); + qm_fd_read(&fd, &p_Dq->fd); if (p_Dq->stat & QM_DQRR_STAT_UNSCHEDULED) { /* We only set QMAN_FQ_STATE_NE when retiring, so we only need * to check for clearing it when doing volatile dequeues. It's * one less thing to check in the critical path (SDQCR). */ - tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, (t_DpaaFD*)&p_Dq->fd); + tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; /* Check for VDQCR completion */ @@ -788,14 +961,14 @@ static void LoopDequeueRingDcaOptimized(t_Handle h_QmPortal) { /* Interpret 'dq' from the owner's perspective. */ /* use portal default handlers */ - ASSERT_COND(p_Dq->fqid); + ASSERT_COND(GET_UINT32(p_Dq->fqid)); if (p_Fq) { tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, - (t_DpaaFD*)&p_Dq->fd); + &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; else if (p_Fq->state == qman_fq_state_waiting_parked) @@ -806,8 +979,8 @@ static void LoopDequeueRingDcaOptimized(t_Handle h_QmPortal) tmpRes = p_QmPortal->p_NullCB->dqrr(p_QmPortal->h_App, NULL, p_QmPortal, - p_Dq->fqid, - (t_DpaaFD*)&p_Dq->fd); + GET_UINT32(p_Dq->fqid), + &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; } @@ -834,6 +1007,7 @@ static void LoopDequeueRingOptimized(t_Handle h_QmPortal) enum qman_cb_dqrr_result res = qman_cb_dqrr_consume; e_RxStoreResponse tmpRes; t_QmPortal *p_QmPortal = (t_QmPortal *)h_QmPortal; + t_DpaaFD fd; while (res != qman_cb_dqrr_pause) { @@ -841,12 +1015,13 @@ static void LoopDequeueRingOptimized(t_Handle h_QmPortal) p_Dq = qm_dqrr_current(p_QmPortal->p_LowQmPortal); if (!p_Dq) break; - p_Fq = ptr_from_aligned_int(p_Dq->contextB); + p_Fq = ptr_from_aligned_int(GET_UINT32(p_Dq->contextB)); + qm_fd_read(&fd, &p_Dq->fd); if (p_Dq->stat & QM_DQRR_STAT_UNSCHEDULED) { /* We only set QMAN_FQ_STATE_NE when retiring, so we only need * to check for clearing it when doing volatile dequeues. It's * one less thing to check in the critical path (SDQCR). */ - tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, (t_DpaaFD*)&p_Dq->fd); + tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; /* Check for VDQCR completion */ @@ -862,14 +1037,14 @@ static void LoopDequeueRingOptimized(t_Handle h_QmPortal) { /* Interpret 'dq' from the owner's perspective. */ /* use portal default handlers */ - ASSERT_COND(p_Dq->fqid); + ASSERT_COND(GET_UINT32(p_Dq->fqid)); if (p_Fq) { tmpRes = p_Fq->cb.dqrr(p_Fq->h_App, p_Fq->h_QmFqr, p_QmPortal, p_Fq->fqidOffset, - (t_DpaaFD*)&p_Dq->fd); + &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; else if (p_Fq->state == qman_fq_state_waiting_parked) @@ -880,8 +1055,8 @@ static void LoopDequeueRingOptimized(t_Handle h_QmPortal) tmpRes = p_QmPortal->p_NullCB->dqrr(p_QmPortal->h_App, NULL, p_QmPortal, - p_Dq->fqid, - (t_DpaaFD*)&p_Dq->fd); + GET_UINT32(p_Dq->fqid), + &fd); if (tmpRes == e_RX_STORE_RESPONSE_PAUSE) res = qman_cb_dqrr_pause; } @@ -909,6 +1084,26 @@ static void portal_isr(void *ptr) DBG(TRACE, ("software-portal %d got interrupt", p_QmPortal->p_LowQmPortal->config.cpu)); +#ifdef __aarch64__ + { + /* Per-portal ISR invocation counters (max 16 portals) */ + static volatile uint32_t portal_isr_cnt[16]; + int cpu = p_QmPortal->p_LowQmPortal->config.cpu; + uint32_t cnt; + if (cpu >= 0 && cpu < 16) + cnt = ++portal_isr_cnt[cpu]; + else + cnt = 0; + if (cnt <= 10 || (cnt % 1000 == 0)) + printf("qman: portal_isr cpu=%d count=%u ISR=0x%x IER=0x%x " + "dqrr_fill=%u\n", + cpu, cnt, + qm_isr_status_read(p_QmPortal->p_LowQmPortal), + enableEvents, + (unsigned)p_QmPortal->p_LowQmPortal->dqrr.fill); + } +#endif + event |= (qm_isr_status_read(p_QmPortal->p_LowQmPortal) & enableEvents); @@ -930,7 +1125,7 @@ static t_Error qman_query_fq_np(t_QmPortal *p_QmPortal, struct qman_fq *p_Fq, st NCSW_PLOCK(p_QmPortal); p_Mcc = qm_mc_start(p_QmPortal->p_LowQmPortal); - p_Mcc->queryfq_np.fqid = p_Fq->fqid; + WRITE_UINT32(p_Mcc->queryfq_np.fqid, p_Fq->fqid); qm_mc_commit(p_QmPortal->p_LowQmPortal, QM_MCC_VERB_QUERYFQ_NP); while (!(p_Mcr = qm_mc_result(p_QmPortal->p_LowQmPortal))) ; ASSERT_COND((p_Mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); @@ -1198,6 +1393,105 @@ static void clean_dqrr_mr(t_QmPortal *p_QmPortal) } } +/* + * Clean up stale frame queues left by a previous boot (e.g. U-Boot). + * The QMan hardware's FQD cache may retain FQs in SCHED/ACTIVE state + * across a soft reboot. If not retired, these stale FQs will produce + * DQRR entries with invalid contextB pointers, causing a page fault. + * + * Called during portal creation after DQRR, MR, and MC are initialized + * but before ISR setup and SDQCR enable. + */ +#define CLEANUP_MAX_FQID 256 + +static void qman_cleanup_stale_fqs(struct qm_portal *portal) +{ + static volatile int cleanup_done = 0; + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + uint32_t fqid; + int cleaned = 0; + uint8_t state, res; + + /* All portals see the same global FQ state — only clean up once. + * Running on subsequent portals would retire FQs that drivers + * legitimately created after the first portal's cleanup. */ + if (atomic_cmpset_int(&cleanup_done, 0, 1) == 0) + return; + + /* Drain any stale DQRR entries from previous boot */ + while (qmPortalDqrrPvbUpdate(portal), + qm_dqrr_current(portal) != NULL) { + qm_dqrr_next(portal); + qmPortalDqrrCciConsume(portal, 1); + } + /* Drain any stale MR entries from previous boot */ + while (qmPortalMrPvbUpdate(portal), + qm_mr_current(portal) != NULL) { + qm_mr_next(portal); + qmPortalMrCciConsume(portal, 1); + } + /* Retire stale FQs via MC commands */ + for (fqid = 0; fqid < CLEANUP_MAX_FQID; fqid++) { + /* QUERYFQ_NP */ + mcc = qm_mc_start(portal); + WRITE_UINT32(mcc->queryfq_np.fqid, fqid); + qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ_NP); + while (!(mcr = qm_mc_result(portal))) + ; + res = mcr->result; + if (res != QM_MCR_RESULT_OK) + continue; + state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; + if (state == QM_MCR_NP_STATE_OOS) + continue; + if (state == QM_MCR_NP_STATE_RETIRED) + goto do_oos; + + /* ALTER_RETIRE */ + mcc = qm_mc_start(portal); + WRITE_UINT32(mcc->alterfq.fqid, fqid); + qm_mc_commit(portal, QM_MCC_VERB_ALTER_RETIRE); + while (!(mcr = qm_mc_result(portal))) + ; + res = mcr->result; + if (res == QM_MCR_RESULT_PENDING) { + /* Wait for FQRNI on MR */ + int timeout = 10000; + while (timeout--) { + qmPortalMrPvbUpdate(portal); + if (qm_mr_current(portal) != NULL) { + qm_mr_next(portal); + qmPortalMrCciConsume(portal, 1); + break; + } + } + } else if (res != QM_MCR_RESULT_OK) { + XX_Print("qman_cleanup: fqid %u retire failed 0x%02x\n", + fqid, res); + continue; + } + +do_oos: + /* ALTER_OOS */ + mcc = qm_mc_start(portal); + WRITE_UINT32(mcc->alterfq.fqid, fqid); + qm_mc_commit(portal, QM_MCC_VERB_ALTER_OOS); + while (!(mcr = qm_mc_result(portal))) + ; + res = mcr->result; + if (res != QM_MCR_RESULT_OK) { + /* OOS may fail if FQ has pending frames. + * RETIRED state is safe — FQ won't be scheduled. */ + XX_Print("qman_cleanup: fqid %u OOS failed 0x%02x " + "(left retired)\n", fqid, res); + } + cleaned++; + } + if (cleaned) + printf("qman: retired %d stale FQs\n", cleaned); +} + static t_Error qman_create_portal(t_QmPortal *p_QmPortal, uint32_t flags, uint32_t sdqcrFlags, @@ -1230,6 +1524,8 @@ static t_Error qman_create_portal(t_QmPortal *p_QmPortal, REPORT_ERROR(MAJOR, E_INVALID_STATE, ("MC initialization failed")); goto fail_mc; } + /* Clean up stale FQs from previous boot before enabling SDQCR */ + qman_cleanup_stale_fqs(p_QmPortal->p_LowQmPortal); if (qm_isr_init(p_QmPortal->p_LowQmPortal)) { REPORT_ERROR(MAJOR, E_INVALID_STATE, ("ISR initialization failed")); goto fail_isr; @@ -1272,7 +1568,37 @@ goto fail_dqrr_mr_empty; goto fail_dqrr_mr_empty; } qm_isr_disable_write(p_QmPortal->p_LowQmPortal, 0); + /* + * Inhibit ISR before enabling SDQCR — stale FQs from a previous boot + * (e.g. Linux) may still be in SCHED state. Enabling SDQCR causes + * the hardware to push their DQRR entries immediately. Drain them + * here before the ISR can see them (their contextB pointers are + * garbage in this kernel's address space). + */ + qm_isr_inhibit(p_QmPortal->p_LowQmPortal); qm_dqrr_sdqcr_set(p_QmPortal->p_LowQmPortal, sdqcrFlags); + { + int idle = 0; + while (idle < 1000) { + qmPortalDqrrPvbUpdate(p_QmPortal->p_LowQmPortal); + if (qm_dqrr_current(p_QmPortal->p_LowQmPortal) != NULL) { + qm_dqrr_next(p_QmPortal->p_LowQmPortal); + qmPortalDqrrCciConsume(p_QmPortal->p_LowQmPortal, 1); + idle = 0; + } else { + XX_UDelay(10); + idle++; + } + } + /* Also drain any MR entries that appeared */ + while (qmPortalMrPvbUpdate(p_QmPortal->p_LowQmPortal), + qm_mr_current(p_QmPortal->p_LowQmPortal) != NULL) { + qm_mr_next(p_QmPortal->p_LowQmPortal); + qmPortalMrCciConsume(p_QmPortal->p_LowQmPortal, 1); + } + } + qm_isr_status_clear(p_QmPortal->p_LowQmPortal, 0xffffffff); + qm_isr_uninhibit(p_QmPortal->p_LowQmPortal); return E_OK; fail_dqrr_mr_empty: fail_eqcr_empty: @@ -1341,8 +1667,8 @@ static t_Error qman_orp_update(t_QmPortal *p_QmPortal, else /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ orpSeqnum &= ~QM_EQCR_SEQNUM_NESN; - p_Eq->seqnum = orpSeqnum; - p_Eq->orp = orpId; + WRITE_UINT16(p_Eq->seqnum, orpSeqnum); + WRITE_UINT32(p_Eq->orp, orpId); qmPortalEqcrPvbCommit(p_QmPortal->p_LowQmPortal, (uint8_t)QM_EQCR_VERB_ORP); PUNLOCK(p_QmPortal); @@ -1394,7 +1720,7 @@ static t_Error QmPortalUnregisterCg(t_Handle h_QmPortal, uint8_t cgId) if(!(p_QmPortal->cgrs[0].q.__state[cgId/32] & (0x80000000 >> (cgId % 32)))) RETURN_ERROR(MINOR, E_BUSY, ("CG is not in use")); - p_QmPortal->cgrs[0].q.__state[cgId/32] &= ~0x80000000 >> (cgId % 32); + p_QmPortal->cgrs[0].q.__state[cgId/32] &= ~(0x80000000 >> (cgId % 32)); p_QmPortal->cgsHandles[cgId] = NULL; return E_OK; @@ -1494,7 +1820,6 @@ static t_Error QmPortalPullFrame(t_Handle h_QmPortal, uint32_t pdqcr, t_DpaaFD * t_QmPortal *p_QmPortal = (t_QmPortal *)h_QmPortal; struct qm_dqrr_entry *p_Dq; int prefetch; - uint32_t *p_Dst, *p_Src; ASSERT_COND(p_QmPortal); ASSERT_COND(p_Frame); @@ -1515,13 +1840,8 @@ static t_Error QmPortalPullFrame(t_Handle h_QmPortal, uint32_t pdqcr, t_DpaaFD * p_Dq = qm_dqrr_current(p_QmPortal->p_LowQmPortal); if (!p_Dq) continue; - ASSERT_COND(p_Dq->fqid); - p_Dst = (uint32_t *)p_Frame; - p_Src = (uint32_t *)&p_Dq->fd; - p_Dst[0] = p_Src[0]; - p_Dst[1] = p_Src[1]; - p_Dst[2] = p_Src[2]; - p_Dst[3] = p_Src[3]; + ASSERT_COND(GET_UINT32(p_Dq->fqid)); + qm_fd_read(p_Frame, &p_Dq->fd); if (p_QmPortal->options & QMAN_PORTAL_FLAG_DCA) { qmPortalDqrrDcaConsume1ptr(p_QmPortal->p_LowQmPortal, @@ -1792,6 +2112,12 @@ t_Error QM_PORTAL_AddPoolChannel(t_Handle h_QmPortal, uint8_t poolChannelId) sdqcrFlags |= QM_SDQCR_CHANNELS_POOL(poolChannelId+1); qm_dqrr_sdqcr_set(p_QmPortal->p_LowQmPortal, sdqcrFlags); +#ifdef __aarch64__ + XX_Print("QM_PORTAL_AddPoolChannel: cpu=%d poolCh=%u SDQCR=0x%08x\n", + p_QmPortal->p_LowQmPortal->config.cpu, + (unsigned)poolChannelId, sdqcrFlags); +#endif + return E_OK; } @@ -1820,6 +2146,51 @@ t_Error QM_PORTAL_Poll(t_Handle h_QmPortal, e_QmPortalPollSource source) return E_OK; } +#ifdef __aarch64__ +/* + * Read DQRR state from hardware for diagnostics. + * Returns software PI/CI/fill and hardware PI/CI registers. + */ +void QM_PORTAL_DqrrDiag(t_Handle h_QmPortal, uint8_t *sw_pi, uint8_t *sw_ci, + uint8_t *sw_fill, uint8_t *hw_pi, uint8_t *hw_ci) +{ + t_QmPortal *p_QmPortal = (t_QmPortal *)h_QmPortal; + struct qm_portal *portal = p_QmPortal->p_LowQmPortal; + struct qm_dqrr *dqrr = &portal->dqrr; + + *sw_pi = dqrr->pi; + *sw_ci = dqrr->ci; + *sw_fill = dqrr->fill; + *hw_pi = (uint8_t)(qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1)); + *hw_ci = (uint8_t)(qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1)); +} + +void QM_PORTAL_IsrDiag(t_Handle h_QmPortal, uint32_t *isr, uint32_t *ier, + uint32_t *iir) +{ + t_QmPortal *p = (t_QmPortal *)h_QmPortal; + + *isr = qm_isr_status_read(p->p_LowQmPortal); + *ier = qm_isr_enable_read(p->p_LowQmPortal); + *iir = __qm_isr_read(p->p_LowQmPortal, qm_isr_inhibit); +} + +void QM_PORTAL_Inhibit(t_Handle h_QmPortal) +{ + t_QmPortal *p = (t_QmPortal *)h_QmPortal; + + qm_isr_inhibit(p->p_LowQmPortal); +} + +void QM_PORTAL_Uninhibit(t_Handle h_QmPortal) +{ + t_QmPortal *p = (t_QmPortal *)h_QmPortal; + + qm_isr_status_clear(p->p_LowQmPortal, 0xffffffff); + qm_isr_uninhibit(p->p_LowQmPortal); +} +#endif + t_Error QM_PORTAL_PollFrame(t_Handle h_QmPortal, t_QmPortalFrameInfo *p_frameInfo) { t_QmPortal *p_QmPortal = (t_QmPortal *)h_QmPortal; @@ -1842,21 +2213,21 @@ t_Error QM_PORTAL_PollFrame(t_Handle h_QmPortal, t_QmPortalFrameInfo *p_frameInf PUNLOCK(p_QmPortal); return ERROR_CODE(E_EMPTY); } - p_Fq = ptr_from_aligned_int(p_Dq->contextB); - ASSERT_COND(p_Dq->fqid); + p_Fq = ptr_from_aligned_int(GET_UINT32(p_Dq->contextB)); + ASSERT_COND(GET_UINT32(p_Dq->fqid)); if (p_Fq) { p_frameInfo->h_App = p_Fq->h_App; p_frameInfo->h_QmFqr = p_Fq->h_QmFqr; p_frameInfo->fqidOffset = p_Fq->fqidOffset; - memcpy((void*)&p_frameInfo->frame, (void*)&p_Dq->fd, sizeof(t_DpaaFD)); + qm_fd_read(&p_frameInfo->frame, &p_Dq->fd); } else { p_frameInfo->h_App = p_QmPortal->h_App; p_frameInfo->h_QmFqr = NULL; - p_frameInfo->fqidOffset = p_Dq->fqid; - memcpy((void*)&p_frameInfo->frame, (void*)&p_Dq->fd, sizeof(t_DpaaFD)); + p_frameInfo->fqidOffset = GET_UINT32(p_Dq->fqid); + qm_fd_read(&p_frameInfo->frame, &p_Dq->fd); } if (p_QmPortal->options & QMAN_PORTAL_FLAG_DCA) { qmPortalDqrrDcaConsume1ptr(p_QmPortal->p_LowQmPortal, @@ -2138,7 +2509,6 @@ t_Error QM_FQR_Enqueue(t_Handle h_QmFqr, t_Handle h_QmPortal, uint32_t fqidOffse t_QmFqr *p_QmFqr = (t_QmFqr *)h_QmFqr; t_QmPortal *p_QmPortal; struct qm_eqcr_entry *p_Eq; - uint32_t *p_Dst, *p_Src; const struct qman_fq *p_Fq; SANITY_CHECK_RETURN_ERROR(p_QmFqr, E_INVALID_HANDLE); @@ -2171,19 +2541,9 @@ t_Error QM_FQR_Enqueue(t_Handle h_QmFqr, t_Handle h_QmPortal, uint32_t fqidOffse return ERROR_CODE(E_BUSY); } - p_Eq->fqid = p_Fq->fqid; - p_Eq->tag = aligned_int_from_ptr(p_Fq); - /* gcc does a dreadful job of the following; - * eq->fd = *fd; - * It causes the entire function to save/restore a wider range of - * registers, and comes up with instruction-waste galore. This will do - * until we can rework the function for better code-generation. */ - p_Dst = (uint32_t *)&p_Eq->fd; - p_Src = (uint32_t *)p_Frame; - p_Dst[0] = p_Src[0]; - p_Dst[1] = p_Src[1]; - p_Dst[2] = p_Src[2]; - p_Dst[3] = p_Src[3]; + WRITE_UINT32(p_Eq->fqid, p_Fq->fqid); + WRITE_UINT32(p_Eq->tag, aligned_int_from_ptr(p_Fq)); + qm_fd_write(&p_Eq->fd, p_Frame); qmPortalEqcrPvbCommit(p_QmPortal->p_LowQmPortal, (uint8_t)(QM_EQCR_VERB_CMD_ENQUEUE/* | @@ -2193,6 +2553,33 @@ t_Error QM_FQR_Enqueue(t_Handle h_QmFqr, t_Handle h_QmPortal, uint32_t fqidOffse return E_OK; } +t_Error QM_PORTAL_EnqueueFqid(t_Handle h_QmPortal, uint32_t fqid, t_DpaaFD *p_Frame) +{ + t_QmPortal *p_QmPortal; + struct qm_eqcr_entry *p_Eq; + + SANITY_CHECK_RETURN_ERROR(h_QmPortal, E_INVALID_HANDLE); + p_QmPortal = (t_QmPortal *)h_QmPortal; + + NCSW_PLOCK(p_QmPortal); + p_Eq = try_eq_start(p_QmPortal); + if (!p_Eq) + { + PUNLOCK(p_QmPortal); + return ERROR_CODE(E_BUSY); + } + + WRITE_UINT32(p_Eq->fqid, fqid); + WRITE_UINT32(p_Eq->tag, 0); + qm_fd_write(&p_Eq->fd, p_Frame); + + qmPortalEqcrPvbCommit(p_QmPortal->p_LowQmPortal, + (uint8_t)QM_EQCR_VERB_CMD_ENQUEUE); + PUNLOCK(p_QmPortal); + + return E_OK; +} + t_Error QM_FQR_PullFrame(t_Handle h_QmFqr, t_Handle h_QmPortal, uint32_t fqidOffset, t_DpaaFD *p_Frame) { diff --git a/sys/contrib/ncsw/Peripherals/QM/qman_low.h b/sys/contrib/ncsw/Peripherals/QM/qman_low.h index 1becfc15554a..687d58f210af 100644 --- a/sys/contrib/ncsw/Peripherals/QM/qman_low.h +++ b/sys/contrib/ncsw/Peripherals/QM/qman_low.h @@ -51,6 +51,30 @@ /***************************/ /* Cache-inhibited register offsets */ +#if defined(__aarch64__) +/* + * ARM64 Layerscape (LS1046A etc.): CI space is 16KB per portal. + * Register offsets differ from PowerPC. + * See Linux drivers/soc/fsl/qbman/qman.c for reference. + */ +#define REG_EQCR_PI_CINH 0x3000 +#define REG_EQCR_CI_CINH 0x3040 +#define REG_EQCR_ITR 0x3080 +#define REG_DQRR_PI_CINH 0x3100 +#define REG_DQRR_CI_CINH 0x3140 +#define REG_DQRR_ITR 0x3180 +#define REG_DQRR_DCAP 0x31C0 +#define REG_DQRR_SDQCR 0x3200 +#define REG_DQRR_VDQCR 0x3240 +#define REG_DQRR_PDQCR 0x3280 +#define REG_MR_PI_CINH 0x3300 +#define REG_MR_CI_CINH 0x3340 +#define REG_MR_ITR 0x3380 +#define REG_CFG 0x3500 +#define REG_ISR 0x3600 +#define REG_ITPR 0x3740 +#define QM_ISR_SHIFT 6 /* ISR/IER/ISDR/IIR stride: 64 bytes */ +#else /* PowerPC */ #define REG_EQCR_PI_CINH 0x0000 #define REG_EQCR_CI_CINH 0x0004 #define REG_EQCR_ITR 0x0008 @@ -66,22 +90,29 @@ #define REG_MR_ITR 0x0088 #define REG_CFG 0x0100 #define REG_ISR 0x0e00 -#define REG_IER 0x0e04 -#define REG_ISDR 0x0e08 -#define REG_IIR 0x0e0c #define REG_ITPR 0x0e14 +#define QM_ISR_SHIFT 2 /* ISR/IER/ISDR/IIR stride: 4 bytes */ +#endif /* Cache-enabled register offsets */ #define CL_EQCR 0x0000 #define CL_DQRR 0x1000 #define CL_MR 0x2000 #define CL_EQCR_PI_CENA 0x3000 +#if defined(__aarch64__) +#define CL_EQCR_CI_CENA 0x3040 +#define CL_DQRR_PI_CENA 0x3100 +#define CL_DQRR_CI_CENA 0x3140 +#define CL_MR_PI_CENA 0x3300 +#define CL_MR_CI_CENA 0x3340 +#else /* PowerPC */ #define CL_EQCR_CI_CENA 0x3100 #define CL_DQRR_PI_CENA 0x3200 #define CL_DQRR_CI_CENA 0x3300 #define CL_MR_PI_CENA 0x3400 #define CL_MR_CI_CENA 0x3500 #define CL_RORI_CENA 0x3600 +#endif #define CL_CR 0x3800 #define CL_RR0 0x3900 #define CL_RR1 0x3940 @@ -300,11 +331,18 @@ static __inline__ struct qm_eqcr_entry *qm_eqcr_pend_and_next(struct qm_portal * } #ifdef QM_CHECKING +/* + * EQCR entries live in CE portal memory (big-endian on hardware). + * On ARM64, WRITE_UINT32 byte-swaps values for the hardware, so we + * must use GET_UINT32 to read them back in host order for validation. + */ #define EQCR_COMMIT_CHECKS(eqcr) \ do { \ ASSERT_COND(eqcr->busy); \ - ASSERT_COND(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff)); \ - ASSERT_COND(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff)); \ + ASSERT_COND(GET_UINT32(eqcr->cursor->orp) == \ + (GET_UINT32(eqcr->cursor->orp) & 0x00ffffff)); \ + ASSERT_COND(GET_UINT32(eqcr->cursor->fqid) == \ + (GET_UINT32(eqcr->cursor->fqid) & 0x00ffffff)); \ } while(0) #else @@ -878,6 +916,7 @@ static __inline__ void qm_mr_current_prefetch(struct qm_portal *portal) { register struct qm_mr *mr = &portal->mr; dcbt_ro(mr->cursor); + (void)mr; } static __inline__ struct qm_mr_entry *qm_mr_current(struct qm_portal *portal) @@ -1139,10 +1178,10 @@ static __inline__ void qm_isr_set_iperiod(struct qm_portal *portal, uint16_t ipe static __inline__ uint32_t __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n) { - return __qm_in(&portal->addr, REG_ISR + (n << 2)); + return __qm_in(&portal->addr, REG_ISR + (n << QM_ISR_SHIFT)); } static __inline__ void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n, uint32_t val) { - __qm_out(&portal->addr, REG_ISR + (n << 2), val); + __qm_out(&portal->addr, REG_ISR + (n << QM_ISR_SHIFT), val); } diff --git a/sys/contrib/ncsw/Peripherals/QM/qman_private.h b/sys/contrib/ncsw/Peripherals/QM/qman_private.h index 0b172879099a..3b5dc9fa7453 100644 --- a/sys/contrib/ncsw/Peripherals/QM/qman_private.h +++ b/sys/contrib/ncsw/Peripherals/QM/qman_private.h @@ -56,6 +56,39 @@ * execution reordering with respect to other code/instructions that manipulate * the same cacheline. */ +#if defined(__aarch64__) +/* + * ARM64 cache operations. + * + * Portal CE region is mapped Normal Non-Cacheable (matching Linux's + * MEMREMAP_WC), so stores go directly to hardware. However, dcbf is + * also used on cacheable DMA memory (FQD, PFDR) in qm.c — there we + * need real cache clean to make CPU writes visible to QMan hardware. + * + * dc cvac (clean by VA to PoC) is the ARM64 equivalent of PPC dcbf. + * On NC addresses it's a harmless no-op; on cached addresses it writes + * dirty lines back to memory. + * + * dc zva MUST NOT be used on portal NC memory — it generates a special + * bus transaction that can hang the interconnect. Use memset instead. + */ +#define lwsync() __asm__ __volatile__ ("dmb ish" : : : "memory") +#define dcbf(addr) do { \ + __asm__ __volatile__ ("dc cvac, %0" : : "r"(addr) : "memory"); \ + __asm__ __volatile__ ("dsb st" : : : "memory"); \ +} while(0) +#define dcbi(p) __asm__ __volatile__ ("dsb sy" : : : "memory") +#define dcbt_ro(addr) do { } while(0) +#define dcbt_rw(addr) do { } while(0) +#define dcbz_64(p) do { memset((p), 0, 64); } while(0) +#define dcbf_64(p) do { \ + __asm__ __volatile__ ("dc cvac, %0" : : "r"(p) : "memory"); \ + __asm__ __volatile__ ("dsb st" : : : "memory"); \ +} while(0) +#define dcbit_ro(p) do { } while(0) + +#else /* PowerPC */ + #define dcbf(addr) \ do { \ __asm__ __volatile__ ("dcbf 0, %0" : : "r" (addr)); \ @@ -132,6 +165,8 @@ #define dcbi(p) dcbf(p) +#endif /* __aarch64__ */ + struct qm_addr { void *addr_ce; /* cache-enabled */ void *addr_ci; /* cache-inhibited */ diff --git a/sys/contrib/ncsw/build/dflags_arm64.h b/sys/contrib/ncsw/build/dflags_arm64.h new file mode 100644 index 000000000000..4a5eec8e0e84 --- /dev/null +++ b/sys/contrib/ncsw/build/dflags_arm64.h @@ -0,0 +1,86 @@ +/*- + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * DPAA1 / NetCommSW build flags for ARM64 (LS1046A). + * + * Equivalent to dflags.h (which targets PowerPC e500mc/P5020). + * Force-included before every ncsw source file via config.dpaa_arm64. + */ +#ifndef DFLAGS_ARM64_H_ +#define DFLAGS_ARM64_H_ + +#include "opt_platform.h" +#include "events_mapping.h" + +/* + * Core selection: ARM64 (Cortex-A72). + * The PPC path uses NCSW_PPC_CORE which pulls in ppc_ext.h and e500v2_ext.h. + * Our ARM path uses NCSW_ARM_CORE which pulls in arm_ext.h with FreeBSD + * ARM64 implementations. + */ +#define NCSW_ARM_CORE +#define NCSW_FREEBSD + +/* Cortex-A72 cache line size (LS1046A) */ +#define CORE_CACHELINE_SIZE 64 + +/* Linux-ism helpers needed by mEMAC flib code */ +#include +#include + +/* + * LIST_ compatibility macros — FreeBSD's list_ext.h prefixes everything + * with NCSW_ to avoid conflicts with . The mEMAC code + * uses the unprefixed original names from the NXP SDK. + * + * Note: LIST_FIRST/NEXT/PREV/LAST conflict with and + * are intentionally omitted here. + */ +#define LIST_Add NCSW_LIST_Add +#define LIST_AddToTail NCSW_LIST_AddToTail +#define LIST_DelAndInit NCSW_LIST_DelAndInit +#define LIST_IsEmpty NCSW_LIST_IsEmpty +#define LIST_OBJECT NCSW_LIST_OBJECT +#define LIST_FOR_EACH NCSW_LIST_FOR_EACH +#define LIST_FOR_EACH_SAFE NCSW_LIST_FOR_EACH_SAFE +#define LIST_FOR_EACH_OBJECT NCSW_LIST_FOR_EACH_OBJECT +#define LIST_FOR_EACH_OBJECT_SAFE NCSW_LIST_FOR_EACH_OBJECT_SAFE + +/* Debugging */ +#define DEBUG_ERRORS 1 +#define DPAA_DEBUG 1 +#if defined(DPAA_DEBUG) +#define DEBUG_GLOBAL_LEVEL REPORT_LEVEL_INFO +#else +#define DEBUG_GLOBAL_LEVEL REPORT_LEVEL_WARNING +#endif + +/* Events */ +#define REPORT_EVENTS 1 +#define EVENT_GLOBAL_LEVEL REPORT_LEVEL_MINOR + +#endif /* DFLAGS_ARM64_H_ */ diff --git a/sys/contrib/ncsw/etc/ncsw_memcpy.c b/sys/contrib/ncsw/etc/ncsw_memcpy.c new file mode 100644 index 000000000000..6793a2ebf845 --- /dev/null +++ b/sys/contrib/ncsw/etc/ncsw_memcpy.c @@ -0,0 +1,6 @@ +/* + * Wrapper to compile ncsw memcpy.c as ncsw_memcpy.o, avoiding + * a name collision with arm64/arm64/memcpy.S (which also produces + * memcpy.o and defines memcpy/memmove). + */ +#include "memcpy.c" diff --git a/sys/contrib/ncsw/inc/Peripherals/bm_ext.h b/sys/contrib/ncsw/inc/Peripherals/bm_ext.h index 952e95f62e47..79a1bff72642 100644 --- a/sys/contrib/ncsw/inc/Peripherals/bm_ext.h +++ b/sys/contrib/ncsw/inc/Peripherals/bm_ext.h @@ -131,6 +131,9 @@ typedef struct { NOTE: this parameter relevant only when working with multiple partitions. */ uint8_t partNumOfPools; /**< Number of Pools dedicated to this partition. NOTE: this parameter relevant only when working with multiple partitions. */ + void *p_FbprBase; /**< Pre-allocated FBPR memory (virtual), or NULL for internal allocation. + Must be physically contiguous, naturally aligned, and non-cacheable. */ + uint32_t fbprSize; /**< Size of pre-allocated FBPR memory in bytes. */ } t_BmParam; diff --git a/sys/contrib/ncsw/inc/Peripherals/dpaa_ext.h b/sys/contrib/ncsw/inc/Peripherals/dpaa_ext.h index e7c0c131fda3..fae846e2b8c6 100644 --- a/sys/contrib/ncsw/inc/Peripherals/dpaa_ext.h +++ b/sys/contrib/ncsw/inc/Peripherals/dpaa_ext.h @@ -182,29 +182,42 @@ typedef _Packed struct t_DpaaCompTbl { #define DPAA_SGTE_BPID_MASK 0x00ff0000 /**< SGTE BPID field mask */ #define DPAA_SGTE_OFFSET_MASK 0x00001fff /**< SGTE OFFSET field mask */ -#define DPAA_SGTE_GET_ADDRH(sgte) (((t_DpaaSGTE *)sgte)->addrh & DPAA_SGTE_ADDRH_MASK) /**< Macro to get SGTE ADDRH field */ -#define DPAA_SGTE_GET_ADDRL(sgte) ((t_DpaaSGTE *)sgte)->addrl /**< Macro to get SGTE ADDRL field */ -#define DPAA_SGTE_GET_PHYS_ADDR(sgte) ((physAddress_t)(((uint64_t)DPAA_SGTE_GET_ADDRH(sgte) << 32) | (uint64_t)DPAA_SGTE_GET_ADDRL(sgte))) /**< Macro to get FD ADDR field */ -#define DPAA_SGTE_GET_EXTENSION(sgte) ((((t_DpaaSGTE *)sgte)->length & DPAA_SGTE_E_MASK) >> (31-0)) /**< Macro to get SGTE EXTENSION field */ -#define DPAA_SGTE_GET_FINAL(sgte) ((((t_DpaaSGTE *)sgte)->length & DPAA_SGTE_F_MASK) >> (31-1)) /**< Macro to get SGTE FINAL field */ -#define DPAA_SGTE_GET_LENGTH(sgte) (((t_DpaaSGTE *)sgte)->length & DPAA_SGTE_LENGTH_MASK) /**< Macro to get SGTE LENGTH field */ -#define DPAA_SGTE_GET_BPID(sgte) ((((t_DpaaSGTE *)sgte)->offset & DPAA_SGTE_BPID_MASK) >> (31-15)) /**< Macro to get SGTE BPID field */ -#define DPAA_SGTE_GET_OFFSET(sgte) (((t_DpaaSGTE *)sgte)->offset & DPAA_SGTE_OFFSET_MASK) /**< Macro to get SGTE OFFSET field */ +/* + * SGT entries live in host DMA memory and are read/written by FMan in + * big-endian byte order. On big-endian hosts (PowerPC) no swap is needed. + * On little-endian hosts (ARM64) each 32-bit field must be byte-swapped. + */ +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define _DPAA_SGTE_R32(field) (field) +#define _DPAA_SGTE_W32(field, val) ((field) = (val)) +#else +#define _DPAA_SGTE_R32(field) __builtin_bswap32(field) +#define _DPAA_SGTE_W32(field, val) ((field) = __builtin_bswap32(val)) +#endif + +#define DPAA_SGTE_GET_ADDRH(sgte) (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->addrh) & DPAA_SGTE_ADDRH_MASK) +#define DPAA_SGTE_GET_ADDRL(sgte) _DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->addrl) +#define DPAA_SGTE_GET_PHYS_ADDR(sgte) ((physAddress_t)(((uint64_t)DPAA_SGTE_GET_ADDRH(sgte) << 32) | (uint64_t)DPAA_SGTE_GET_ADDRL(sgte))) +#define DPAA_SGTE_GET_EXTENSION(sgte) ((_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->length) & DPAA_SGTE_E_MASK) >> (31-0)) +#define DPAA_SGTE_GET_FINAL(sgte) ((_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->length) & DPAA_SGTE_F_MASK) >> (31-1)) +#define DPAA_SGTE_GET_LENGTH(sgte) (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->length) & DPAA_SGTE_LENGTH_MASK) +#define DPAA_SGTE_GET_BPID(sgte) ((_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->offset) & DPAA_SGTE_BPID_MASK) >> (31-15)) +#define DPAA_SGTE_GET_OFFSET(sgte) (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->offset) & DPAA_SGTE_OFFSET_MASK) #define DPAA_SGTE_GET_ADDR(sgte) XX_PhysToVirt(DPAA_SGTE_GET_PHYS_ADDR(sgte)) -#define DPAA_SGTE_SET_ADDRH(sgte,val) (((t_DpaaSGTE *)sgte)->addrh = ((((t_DpaaSGTE *)sgte)->addrh & ~DPAA_SGTE_ADDRH_MASK) | ((val) & DPAA_SGTE_ADDRH_MASK))) /**< Macro to set SGTE ADDRH field */ -#define DPAA_SGTE_SET_ADDRL(sgte,val) ((t_DpaaSGTE *)sgte)->addrl = (val) /**< Macro to set SGTE ADDRL field */ +#define DPAA_SGTE_SET_ADDRH(sgte,val) _DPAA_SGTE_W32(((t_DpaaSGTE *)sgte)->addrh, (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->addrh) & ~DPAA_SGTE_ADDRH_MASK) | ((val) & DPAA_SGTE_ADDRH_MASK)) +#define DPAA_SGTE_SET_ADDRL(sgte,val) _DPAA_SGTE_W32(((t_DpaaSGTE *)sgte)->addrl, (val)) #define DPAA_SGTE_SET_ADDR(sgte,val) \ do { \ uint64_t physAddr = (uint64_t)(XX_VirtToPhys(val)); \ DPAA_SGTE_SET_ADDRH(sgte, ((uint32_t)(physAddr >> 32))); \ DPAA_SGTE_SET_ADDRL(sgte, (uint32_t)physAddr); \ -} while (0) /**< Macro to set SGTE ADDR field */ -#define DPAA_SGTE_SET_EXTENSION(sgte,val) (((t_DpaaSGTE *)sgte)->length = ((((t_DpaaSGTE *)sgte)->length & ~DPAA_SGTE_E_MASK) | (((val) << (31-0))& DPAA_SGTE_E_MASK))) /**< Macro to set SGTE EXTENSION field */ -#define DPAA_SGTE_SET_FINAL(sgte,val) (((t_DpaaSGTE *)sgte)->length = ((((t_DpaaSGTE *)sgte)->length & ~DPAA_SGTE_F_MASK) | (((val) << (31-1))& DPAA_SGTE_F_MASK))) /**< Macro to set SGTE FINAL field */ -#define DPAA_SGTE_SET_LENGTH(sgte,val) (((t_DpaaSGTE *)sgte)->length = (((t_DpaaSGTE *)sgte)->length & ~DPAA_SGTE_LENGTH_MASK) | ((val) & DPAA_SGTE_LENGTH_MASK)) /**< Macro to set SGTE LENGTH field */ -#define DPAA_SGTE_SET_BPID(sgte,val) (((t_DpaaSGTE *)sgte)->offset = ((((t_DpaaSGTE *)sgte)->offset & ~DPAA_SGTE_BPID_MASK) | (((val) << (31-15))& DPAA_SGTE_BPID_MASK))) /**< Macro to set SGTE BPID field */ -#define DPAA_SGTE_SET_OFFSET(sgte,val) (((t_DpaaSGTE *)sgte)->offset = ((((t_DpaaSGTE *)sgte)->offset & ~DPAA_SGTE_OFFSET_MASK) | (((val) << (31-31))& DPAA_SGTE_OFFSET_MASK) )) /**< Macro to set SGTE OFFSET field */ +} while (0) +#define DPAA_SGTE_SET_EXTENSION(sgte,val) _DPAA_SGTE_W32(((t_DpaaSGTE *)sgte)->length, (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->length) & ~DPAA_SGTE_E_MASK) | (((val) << (31-0)) & DPAA_SGTE_E_MASK)) +#define DPAA_SGTE_SET_FINAL(sgte,val) _DPAA_SGTE_W32(((t_DpaaSGTE *)sgte)->length, (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->length) & ~DPAA_SGTE_F_MASK) | (((val) << (31-1)) & DPAA_SGTE_F_MASK)) +#define DPAA_SGTE_SET_LENGTH(sgte,val) _DPAA_SGTE_W32(((t_DpaaSGTE *)sgte)->length, (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->length) & ~DPAA_SGTE_LENGTH_MASK) | ((val) & DPAA_SGTE_LENGTH_MASK)) +#define DPAA_SGTE_SET_BPID(sgte,val) _DPAA_SGTE_W32(((t_DpaaSGTE *)sgte)->offset, (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->offset) & ~DPAA_SGTE_BPID_MASK) | (((val) << (31-15)) & DPAA_SGTE_BPID_MASK)) +#define DPAA_SGTE_SET_OFFSET(sgte,val) _DPAA_SGTE_W32(((t_DpaaSGTE *)sgte)->offset, (_DPAA_SGTE_R32(((t_DpaaSGTE *)sgte)->offset) & ~DPAA_SGTE_OFFSET_MASK) | (((val) << (31-31)) & DPAA_SGTE_OFFSET_MASK)) /* @} */ #if defined(__MWERKS__) && !defined(__GNUC__) diff --git a/sys/contrib/ncsw/inc/Peripherals/fm_ehash.h b/sys/contrib/ncsw/inc/Peripherals/fm_ehash.h new file mode 100644 index 000000000000..b2c5db4f1065 --- /dev/null +++ b/sys/contrib/ncsw/inc/Peripherals/fm_ehash.h @@ -0,0 +1,738 @@ +/* + * Shadow fm_ehash.h — FMan External Hash Table API + * + * Replaces the SDK header (which contains Linux-specific inline functions). + * Struct layouts match the NXP SDK exactly: + * sdk_fman/inc/Peripherals/fm_ehash.h + * + * NOTE: en_exthash_info and en_exthash_node are NOT defined here — they + * are defined locally in fm_ehash_freebsd.c (EXCLUDE_FMAN_IPR_OFFLOAD + * variant). CDX code treats table handles as opaque void*. + * + * Copyright 2011, 2014 Freescale Semiconductor, Inc. + * Copyright (c) 2026 Mono Technologies Inc. + * SPDX-License-Identifier: BSD-2-Clause + */ +#ifndef FM_EHASH_H +#define FM_EHASH_H 1 + +#include + +/* ---------------------------------------------------------------- + * Constants + * ---------------------------------------------------------------- */ +#define MAX_KEY_LEN 56 +#define MAX_EN_EHASH_EXT_ENTRY_SIZE 320 +#define MAX_EN_EHASH_ENTRY_SIZE 256 +#define EN_EHASH_ENTRY_ALIGN 256 +#define TBLENTRY_OPC_ALIGN sizeof(uint32_t) +#define MAX_OPCODES 16 + +/* ---------------------------------------------------------------- + * Entry flag accessors + * ---------------------------------------------------------------- */ +#define SET_INVALID_ENTRY_64BIT(entry) (entry |= (((uint64_t)1) << 63)) +#define SET_INVALID_ENTRY(flags) (flags |= (1 << 15)) +#define SET_TIMESTAMP_ENABLE(flags) (flags |= (1 << 13)) +#define SET_STATS_ENABLE(flags) (flags |= (1 << 12)) +#define SET_OPC_OFFSET(flags, offset) (flags |= ((offset >> 2) << 6)) +#define SET_PARAM_OFFSET(flags, offset) (flags |= (offset >> 2)) + +#define GET_INVALID_ENTRY_64BIT(entry) (entry & (((uint64_t)1) << 63)) +#define GET_INVALID_ENTRY(flags) (flags & (1 << 15)) +#define GET_TIMESTAMP_ENABLE(flags) ((flags >> 13) & 1) +#define GET_STATS_ENABLE(flags) (flags & (1 << 12)) +#define GET_OPC_OFFSET(x) (((x >> 6) & 0x1f) << 2) +#define GET_PARAM_OFFSET(x) ((x & 0x3f) << 2) + +/* ---------------------------------------------------------------- + * Hash table entry — on-hardware representation + * ---------------------------------------------------------------- */ +struct en_ehash_entry { + union { + struct { + union { + struct { + uint16_t flags; + uint16_t next_entry_hi; + uint32_t next_entry_lo; + }; + uint64_t next_entry; + }; + uint8_t key[0]; + } __attribute__((packed)); + struct { + uint8_t hash_entry[MAX_EN_EHASH_ENTRY_SIZE]; + uint64_t packet_count; + uint64_t packet_bytes; + uint32_t timestamp; + uint32_t reserved; + uint32_t timestamp_counter; + } __attribute__((packed)); + uint8_t hash_ext_entry[MAX_EN_EHASH_EXT_ENTRY_SIZE]; + } __attribute__((packed)); +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * DPA Classifier table entry statistics + * ---------------------------------------------------------------- */ +#define STATS_VALID (1 << 0) +#define TIMESTAMP_VALID (1 << 1) + +struct en_tbl_entry_stats { + uint64_t pkts; + uint64_t bytes; + uint32_t timestamp; + uint32_t flags; +}; + +/* ---------------------------------------------------------------- + * Opcodes + * ---------------------------------------------------------------- */ +#define ENQUEUE_PKT 0x01 +#define REPLICATE_PKT 0x02 +#define ENQUEUE_ONLY 0x03 +#define UPDATE_ETH_RX_STATS 0x04 +#define PREEMPTIVE_CHECKS_ON_PKT 0x05 +#define PREEMPTIVE_CHECKS_ON_IPSEC_PKT 0x06 +#define STRIP_ETH_HDR 0x11 +#define STRIP_ALL_VLAN_HDRS 0x12 +#define STRIP_PPPoE_HDR 0x14 +#define STRIP_L2_HDR 0x17 +#define STRIP_FIRST_VLAN_HDR 0x18 +#define REMOVE_FIRST_IP_HDR 0x19 +#define VALIDATE_IPSEC_ID 0x1a +#define UPDATE_TTL 0x21 +#define UPDATE_SIP_V4 0x22 +#define UPDATE_DIP_V4 0x24 +#define UPDATE_HOPLIMIT 0x29 +#define UPDATE_SIP_V6 0x2A +#define UPDATE_DIP_V6 0x2C +#define UPDATE_SPORT 0x31 +#define UPDATE_DPORT 0x32 +#define INSERT_L2_HDR 0x41 +#define INSERT_VLAN_HDR 0x42 +#define INSERT_PPPoE_HDR 0x43 +#define INSERT_L3_HDR 0x44 +#define REPLACE_PPPOE_HDR 0x45 +#define NATPT_4to6 0x51 +#define NATPT_6to4 0x52 +#define PROCESS_RTP_PAYLOAD 0x61 +#define PROCESS_RTCP_PAYLOAD 0x62 +#define UPDATE_GLOBAL_STATS 0x80 + +/* ---------------------------------------------------------------- + * Preemptive check parameters + * ---------------------------------------------------------------- */ +#define PREEMPT_TX_VALIDATE (1 << 0) +#define PREEMPT_DFBIT_HONOR (1 << 1) +#define PREEMPT_POLICE_PKT (1 << 2) +#define PREEMPT_MATCH_DSCP (1 << 3) +#define PREEMPT_REPLACE_DSCP (1 << 4) + +struct en_ehash_preempt_op { + uint8_t mtu_offset; + uint8_t OpMask; + uint8_t dscp_match_value; + uint8_t pp_no; + uint8_t new_dscp_val; + uint8_t pad; + uint16_t pad1; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * IPSec preemptive check parameters + * ---------------------------------------------------------------- */ +#define MAX_VLAN_PER_FLOW 2 +#define MAX_SPI_PER_FLOW 16 +#define VALIDATE_SPI (0x1 << 0) + +struct spi_info { + uint32_t spi; + uint32_t fqid; +} __attribute__((packed)); + +struct en_ehash_ipsec_preempt_op { + uint8_t op_flags; + uint8_t unused; + uint16_t natt_arr_mask; + uint16_t pppoe_session_id; + uint16_t pad; + struct spi_info spi_param[MAX_SPI_PER_FLOW]; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Enqueue parameters + * ---------------------------------------------------------------- */ +#define EN_EHASH_DISABLE_FRAG 0xffff + +struct en_ehash_enqueue_param { + uint16_t mtu; + uint8_t hdr_xpnd_sz; + uint8_t bpid; + uint32_t fqid; + union { + struct { + uint32_t rspid:8; + uint32_t stats_ptr:24; + }; + uint32_t word; + }; + union { + struct { + uint32_t dscp_fq_enable:8; + uint32_t muram_frag_param_addr:24; + }; + uint32_t word2; + }; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * RTP relay parameters + * ---------------------------------------------------------------- */ +#define EEH_RTP_SEND_FIRST_PACKET_TO_CP 0x0001 +#define EEH_RTP_DUPLICATE_PKT_SEND_TO_CP 0x0002 +#define EEH_RTP_ENABLE_VLAN_P_BIT_LEARN 0x0004 + +struct en_ehash_rtprelay_param { + uint32_t rtpinfo_ptr; + uint32_t in_sock_stats_ptr; + uint32_t out_sock_stats_ptr; + union { + uint32_t src_ipv4_val; + uint32_t src_ipv6_val[4]; + } __attribute__((packed)); + uint32_t TimeStampIncr; + uint32_t SSRC_1; + uint16_t seq_base; + uint16_t egress_socketID; + uint8_t DTMF_PT[2]; + uint16_t rtp_flags; + uint16_t seq_incr; + uint32_t chksum_ptr; + uint32_t rtp_hdr; + uint32_t ts_incr; + uint32_t cur_ts_msec; + int32_t rtp_check; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Replicate parameters (multicast) + * ---------------------------------------------------------------- */ +struct en_ehash_replicate_param { + union { + struct { + uint16_t rsvd; + uint16_t first_member_flow_addr_hi; + uint32_t first_member_flow_addr_lo; + }; + uint64_t first_member_flow_addr; + }; + void *first_listener_entry; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Update ethernet RX stats + * ---------------------------------------------------------------- */ +struct en_ehash_update_ether_rx_stats { + uint32_t stats_ptr; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Strip first VLAN header + * ---------------------------------------------------------------- */ +struct en_ehash_strip_first_vlan_hdr { + uint32_t stats_ptr; + uint16_t vlan_id; + uint16_t pad; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Strip all VLAN headers + * ---------------------------------------------------------------- */ +#define OP_SKIP_VLAN_VALIDATE (1 << 0) +#define OP_VLAN_FILTER_EN (1 << 1) +#define OP_VLAN_FILTER_PVID_SET (1 << 2) + +struct en_ehash_strip_all_vlan_hdrs { + uint16_t vlan_id[MAX_VLAN_PER_FLOW]; + union { + struct { + uint32_t padding:2; + uint32_t num_entries:6; + uint32_t stats_ptr:24; + }; + uint32_t word; + }; + uint8_t op_flags; + uint8_t pad; + uint16_t pad1; + uint8_t stats_offsets[0]; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Strip PPPoE header + * ---------------------------------------------------------------- */ +struct en_ehash_strip_pppoe_hdr { + uint32_t stats_ptr; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Strip all L2 headers + * ---------------------------------------------------------------- */ +struct en_ehash_strip_l2_hdrs { + uint16_t vlan_id[MAX_VLAN_PER_FLOW]; + union { + struct { + uint32_t padding:2; + uint32_t num_entries:6; + uint32_t stats_ptr:24; + }; + uint32_t word; + }; + uint8_t stats_offsets[0]; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Validate IPSec ID + * ---------------------------------------------------------------- */ +struct en_ehash_validate_ipsec { + uint32_t reserved:16; + uint32_t identifier:16; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Update IP addresses + * ---------------------------------------------------------------- */ +struct en_ehash_update_ipv4_ip { + uint32_t ip_v4; +} __attribute__((packed)); + +struct en_ehash_update_ipv6_ip { + uint8_t ip_v6[16]; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Update DSCP + * ---------------------------------------------------------------- */ +struct en_ehash_update_dscp { + union { + struct { + uint32_t rsvd:24; + uint32_t dscp_mark_value:6; + uint32_t dscp_mark_flag:1; + uint32_t padding:1; + }; + uint32_t dscp; + }; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Update ports + * ---------------------------------------------------------------- */ +struct en_ehash_update_port { + uint16_t dport; + uint16_t sport; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Insert L2 header + * ---------------------------------------------------------------- */ +struct en_ehash_insert_l2_hdr { + union { + struct { + uint8_t replace:1; + uint8_t header_padding:2; + uint8_t reserved:5; + uint8_t stats_count; + uint8_t reserved_1; + uint8_t hdr_len; + }; + uint32_t word; + }; + uint8_t l2hdr[0]; +} __attribute__((packed)); + +struct en_ehash_insert_l2_hdr_stats { + union { + struct { + uint32_t padding:2; + uint32_t reserved:6; + uint32_t stats_ptr:24; + }; + uint32_t word; + }; + uint8_t stats_offsets[0]; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Insert VLAN header + * ---------------------------------------------------------------- */ +struct en_ehash_insert_vlan_hdr { + union { + struct { + uint32_t reserved:1; + uint32_t dscp_vlanpcp_map_enable:1; + uint32_t num_hdrs:6; + uint32_t statptr:24; + } __attribute__((packed)); + uint32_t word; + } __attribute__((packed)); + uint32_t vlanhdr[0]; +} __attribute__((packed)); + +struct en_ehash_insert_vlan_hdr_stats { + uint8_t stats_offsets[1]; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Insert/replace PPPoE header + * ---------------------------------------------------------------- */ +#define PPPoE_VERSION 1 +#define PPPoE_TYPE 1 +#define PPPoE_CODE 0 + +struct en_ehash_insert_pppoe_hdr { + uint32_t stats_ptr; + union { + struct { + uint32_t version:4; + uint32_t type:4; + uint32_t code:8; + uint32_t session_id:16; + }; + uint32_t word; + }; +} __attribute__((packed)); + +struct en_ehash_replace_pppoe_hdr_params { + uint8_t destination_mac[6]; + uint8_t source_mac[6]; + uint16_t session_id; + uint16_t pad; + uint32_t fqid; + uint32_t stats_ptr; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Insert L3 header (tunnels) + * ---------------------------------------------------------------- */ +#define TYPE_CUSTOM 0 +#define TYPE_4o6 1 +#define TYPE_6o4 2 + +#define IPID_STARTVAL 1 + +struct en_ehash_insert_l3_hdr { + union { + struct { + uint8_t reserved:3; + uint8_t calc_cksum:1; + uint8_t qos:1; + uint8_t df:1; + uint8_t type:2; + uint8_t hdr_len; + uint16_t ipident; + }; + uint32_t word; + }; + union { + struct { + uint32_t route_dest_offset:8; + uint32_t stats_ptr:24; + }; + uint32_t word_1; + }; + uint8_t l3hdr[0]; +} __attribute__((packed)); + +#define COPY_DSCP_OUTER_INNER (1 << 24) + +/* ---------------------------------------------------------------- + * Remove first IP header (tunnel decap) + * ---------------------------------------------------------------- */ +struct en_ehash_remove_first_ip_hdr { + uint32_t flags:8; + uint32_t stats_ptr:24; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * Port info / interface stats structures + * ---------------------------------------------------------------- */ +struct en_ehash_portinfo { + uint32_t reserved[3]; + uint32_t port_info; +} __attribute__((packed)); + +struct en_ehash_ifportinfo { + struct en_ehash_portinfo rxpinfo; + struct en_ehash_portinfo txpinfo; +} __attribute__((packed)); + +struct en_ehash_stats { + uint64_t bytes; + uint32_t pkts; + uint32_t reserved; +} __attribute__((packed)); + +struct en_ehash_stats_with_ts { + uint64_t bytes; + uint32_t pkts; + uint32_t pad; + uint64_t timestamp; +} __attribute__((packed)); + +struct en_ehash_ifstats { + struct en_ehash_stats rxstats; + struct en_ehash_stats txstats; +} __attribute__((packed)); + +#define STATS_WITH_TS (1 << 7) + +struct en_ehash_ifstats_with_ts { + struct en_ehash_stats_with_ts rxstats; + struct en_ehash_stats_with_ts txstats; +} __attribute__((packed)); + +/* Statistics table */ +struct en_ehash_stats_tbl { + uint32_t table_indicator:1; + uint32_t num_entries:7; + uint32_t stats_ptr:24; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * NATPT structures + * ---------------------------------------------------------------- */ +struct en_ehash_natpt_hdr { + union { + struct { + uint32_t reserved:6; + uint32_t tcu:1; + uint32_t hlu:1; + uint32_t hdrlen:8; + uint32_t reserved1:16; + } ip4to6; + struct { + uint16_t reserved:5; + uint16_t ipu:1; + uint16_t tou:1; + uint16_t tlu:1; + uint16_t hdrlen:8; + uint16_t ipident; + } ip6to4; + uint32_t word; + }; + uint8_t l3hdr[0]; +} __attribute__((packed)); + +/* NATPT 4-to-6 flags */ +#define NATPT_TCU (1 << 25) +#define NATPT_HLU (1 << 24) +/* NATPT 6-to-4 flags */ +#define NATPT_IPU (1 << 26) +#define NATPT_TOU (1 << 25) +#define NATPT_TLU (1 << 24) + +/* ---------------------------------------------------------------- + * SEC failure statistics + * ---------------------------------------------------------------- */ +typedef struct en_SEC_failure_stats_s { + uint32_t icv_failures; + uint32_t hw_errs; + uint32_t CCM_AAD_size_errs; + uint32_t anti_replay_late_errs; + uint32_t anti_replay_replay_errs; + uint32_t seq_num_overflows; + uint32_t DMA_errs; + uint32_t DECO_watchdog_timer_timedout_errs; + uint32_t input_frame_read_errs; + uint32_t protocol_format_errs; + uint32_t ipsec_ttl_zero_errs; + uint32_t ipsec_pad_chk_failures; + uint32_t output_frame_length_rollover_errs; + uint32_t tbl_buff_too_small_errs; + uint32_t tbl_buff_pool_depletion_errs; + uint32_t output_frame_too_large_errs; + uint32_t cmpnd_frame_write_errs; + uint32_t buff_too_small_errs; + uint32_t buff_pool_depletion_errs; + uint32_t output_frame_write_errs; + uint32_t cmpnd_frame_read_errs; + uint32_t prehdr_read_errs; + uint32_t other_errs; +} __attribute__((packed)) en_SEC_failure_stats; + +/* ---------------------------------------------------------------- + * DSCP-to-VLAN PCP mapping + * ---------------------------------------------------------------- */ +#define MAX_VLAN_PCP 7 + +typedef struct en_dscp_vlanpcp_map_cfg_s { + uint8_t dscp_vlanpcp[MAX_VLAN_PCP + 1]; +} __attribute__((packed)) en_dscp_vlanpcp_map_cfg; + +/* ---------------------------------------------------------------- + * Global MURAM data area — at pool + EN_INTERNAL_BUFF_POOL_SIZE + * + * Contains DSCP-to-VLAN PCP mapping (8 bytes) followed by SEC + * failure statistics (92 bytes). Total: 100 bytes within a + * 256-byte MURAM slot. CDX microcode reads/writes these at runtime. + * ---------------------------------------------------------------- */ +typedef struct en_exthash_global_mem_s { + en_dscp_vlanpcp_map_cfg dscp_vlanpcp_map; /* +0, 8B */ + en_SEC_failure_stats SEC_failure_stats; /* +8, 92B */ +} __attribute__((packed)) en_exthash_global_mem; + +/* ---------------------------------------------------------------- + * IP reassembly MURAM parameters — written by FMan microcode, + * fields configured by ExternalHashSetReasslyPool(). + * + * This is the hardware layout in MURAM, pointed to by + * en_exthash_node.word_2 (reassm_param << 8) for REASSM tables. + * All multi-byte fields are stored big-endian. + * ---------------------------------------------------------------- */ +#define MAX_REASSM_BUCKETS 16 + +struct ip_reassembly_stats { + uint64_t num_frag_pkts; + uint64_t num_reassemblies; + uint64_t num_completed_reassly; + uint64_t num_sess_matches; + uint64_t num_frags_too_small; + uint64_t num_reassm_timeouts; + uint64_t num_overlapping_frags; + uint64_t num_too_many_frags; + uint64_t num_failed_bufallocs; + uint64_t num_failed_ctxallocs; + uint64_t num_fatal_errors; + uint64_t num_failed_ctxdeallocs; + uint32_t reassm_count; + uint32_t pad; +} __attribute__((packed)); + +struct ip_reassembly_params { + uint32_t table_base_hi; + uint32_t table_base_lo; + struct ip_reassembly_stats stats; + uint32_t table_mask; + uint32_t type; + uint32_t ipr_timer; + uint32_t timeout_val; + uint32_t timeout_fqid; + uint32_t min_frag_size; + uint32_t reassem_bpid; + uint32_t reassem_bsize; + uint32_t frag_bpid; + uint32_t frag_bsize; + uint32_t reassly_dbg; + uint32_t context_info; + uint32_t curr_sessions; + uint32_t txc_fqid; + uint32_t timer_tnum; + uint32_t max_frags; + uint32_t max_con_reassm; + uint32_t bucket_base; + uint32_t bucket_lock[MAX_REASSM_BUCKETS]; + uint32_t bucket_head[MAX_REASSM_BUCKETS]; +} __attribute__((packed)); + +/* ---------------------------------------------------------------- + * IP reassembly info (for devman.h forward reference) + * ---------------------------------------------------------------- */ +struct ip_reassembly_info { + uint64_t num_frag_pkts; + uint64_t num_reassemblies; + uint64_t num_completed_reassly; + uint64_t num_sess_matches; + uint64_t num_frags_too_small; + uint64_t num_reassm_timeouts; + uint64_t num_overlapping_frags; + uint64_t num_too_many_frags; + uint64_t num_failed_bufallocs; + uint64_t num_failed_ctxallocs; + uint64_t num_fatal_errors; + uint64_t num_failed_ctxdeallocs; + uint32_t table_mask; + uint32_t ipr_timer; + uint32_t timeout_val; + uint32_t timeout_fqid; + uint32_t max_frags; + uint32_t min_frag_size; + uint32_t max_con_reassm; + uint32_t reassem_bpid; + uint32_t reassem_bsize; + uint32_t frag_bpid; + uint32_t frag_bsize; + uint32_t timer_tnum; + uint32_t reassly_dbg; + uint32_t curr_sessions; + uint32_t txc_fqid; +}; + +/* ---------------------------------------------------------------- + * Software table entry wrapping the hardware entry + * ---------------------------------------------------------------- */ +struct en_exthash_tbl_entry { + struct en_ehash_entry hashentry; + struct en_exthash_tbl_entry *prev; + struct en_exthash_tbl_entry *next; + uint8_t *replicate_params; + union { + uint8_t *enqueue_params; + uint8_t *ipsec_preempt_params; + }; +}; + +/* Hash table bucket */ +struct en_exthash_bucket { + uint64_t h; + uint64_t pad; +}; + +/* ---------------------------------------------------------------- + * Debug display — no-op stubs replacing SDK inline functions + * that use printk/phys_to_virt/etc. Tier 1 code calls these + * but they're debug-only and not needed for correct operation. + * ---------------------------------------------------------------- */ +static inline void +display_ehash_tbl_entry(struct en_ehash_entry *entry __attribute__((unused)), + uint32_t keysize __attribute__((unused))) +{ +} + +/* ---------------------------------------------------------------- + * ExternalHash API + * ---------------------------------------------------------------- */ +extern void *ExternalHashTableAllocEntry(void *h_HashTbl); +extern void ExternalHashTableEntryFree(void *entry); +extern int ExternalHashTableFmPcdHcSync(void *h_HashTbl); +extern int ExternalHashTableAddKey(void *h_HashTbl, uint8_t keySize, + void *tbl_entry); +extern int ExternalHashTableDeleteKey(void *h_HashTbl, uint16_t index, + void *tbl_entry); +extern int ExternalHashTableEntryGetStatsAndTS(void *tbl_entry, + struct en_tbl_entry_stats *stats); +extern int32_t ExternalHashGetSECfailureStats(en_SEC_failure_stats *stats); +extern int32_t ExternalHashResetSECfailureStats(void); +extern int32_t ExternalHashSetDscpVlanpcpMapCfg(en_dscp_vlanpcp_map_cfg *map); +extern int32_t ExternalHashGetDscpVlanpcpMapCfg(en_dscp_vlanpcp_map_cfg *map); +extern int ExternalHashSetReasslyPool(uint32_t type, uint32_t ctx_bpid, + uint32_t ctx_bpsize, uint32_t frag_bpid, uint32_t frag_size, + uint32_t txc_fqid, uint32_t ipr_timer_freq); + +/* Kernel-internal ehash API (called from fm_cc.c) */ +extern t_Handle ExternalHashTableSet(t_Handle h_FmPcd, + t_FmPcdHashTableParams *p_Param); +extern void copy_td_to_ccbase(void *handle, t_Handle p_CcTreeTmp); +extern t_Error ExternalHashTableModifyMissNextEngine(t_Handle h_HashTbl, + t_FmPcdCcNextEngineParams *p_FmPcdCcNextEngineParams); +extern void ExternalHashTableDelete(t_Handle h_HashTbl); +extern int ExternalHashReasslyTableExists(uint32_t type); + +/* CDX module exports */ +extern void ipr_update_timestamp(void); +extern void cdx_ehash_update_timestamp(uint32_t id, uint32_t value); +extern uint32_t cdx_ehash_get_timestamp_addr(uint32_t id); + +#endif /* FM_EHASH_H */ diff --git a/sys/contrib/ncsw/inc/Peripherals/fm_ext.h b/sys/contrib/ncsw/inc/Peripherals/fm_ext.h index c3633592694c..d12600c89bf3 100644 --- a/sys/contrib/ncsw/inc/Peripherals/fm_ext.h +++ b/sys/contrib/ncsw/inc/Peripherals/fm_ext.h @@ -1462,6 +1462,11 @@ t_Error FM_GetFmanCtrlCodeRevision(t_Handle h_Fm, t_FmCtrlCodeRevisionInfo *p_Re *//***************************************************************************/ uint32_t FM_GetCounter(t_Handle h_Fm, e_FmCounters counter); +#ifdef __aarch64__ +void FM_ErrorDiag(t_Handle h_Fm, uint8_t hwPortId); +void FM_TaskStatusDiag(t_Handle h_Fm, uint8_t hwPortId); +#endif + /**************************************************************************//** @Function FM_ModifyCounter diff --git a/sys/contrib/ncsw/inc/Peripherals/fm_pcd_ext.h b/sys/contrib/ncsw/inc/Peripherals/fm_pcd_ext.h index 8d1c3d889451..488b4c81d792 100644 --- a/sys/contrib/ncsw/inc/Peripherals/fm_pcd_ext.h +++ b/sys/contrib/ncsw/inc/Peripherals/fm_pcd_ext.h @@ -39,6 +39,10 @@ #ifndef __FM_PCD_EXT #define __FM_PCD_EXT +/* CDX enhanced external hash tables — redirect FM_PCD_HashTable* APIs + * to ExternalHashTable* implementations (matching Linux SDK). */ +#define USE_ENHANCED_EHASH 1 + #include "std_ext.h" #include "net_ext.h" #include "list_ext.h" @@ -1718,6 +1722,8 @@ typedef struct t_FmPcdKgSchemeParams { uint8_t relativeSchemeId; /**< if modify=FALSE:Partition relative scheme id */ t_Handle h_Scheme; /**< if modify=TRUE: a handle of the existing scheme */ } id; + bool shared; /**< This scheme is shared between ports + (must match ioc_fm_pcd_kg_scheme_params_t layout) */ bool alwaysDirect; /**< This scheme is reached only directly, i.e. no need for match vector; KeyGen will ignore it when matching */ struct { /**< HL Relevant only if alwaysDirect = FALSE */ @@ -1991,6 +1997,34 @@ typedef struct t_FmPcdHashTableParams { t_FmPcdCcNextEngineParams ccNextEngineParamsForMiss; /**< Parameters for defining the next engine when a key is not matched */ + bool agingSupport; /**< TRUE to enable aging support for all keys of this + hash table; allows monitoring whether a key was + accessed during a given period. */ + +#if (DPAA_VERSION >= 11) + bool externalHash; /**< TRUE to use external hash table (CDX manages + the hash buckets in DDR; NCSW only allocates the + MURAM AD placeholder). */ + +#ifndef EXCLUDE_FMAN_IPR_OFFLOAD + uint32_t table_type; /**< IP reassembly table type + (must match ioc_fm_pcd_hash_table_params_t layout) */ + struct { + uint32_t timeout_val; /**< reassembly timeout */ + uint32_t timeout_fqid; /**< fqid for reassembly failures */ + uint32_t max_frags; /**< max allowed fragments */ + uint32_t min_frag_size; /**< min allowed frag size except last frag */ + uint32_t max_sessions; /**< max conn reassembly sessions */ + }; +#endif /* EXCLUDE_FMAN_IPR_OFFLOAD */ + + struct { + uint8_t dataMemId; /**< Memory partition ID for external hash table + buckets and contexts; must not cross 4GB boundary */ + uint16_t dataLiodnOffs; /**< LIODN offset for external hash table access */ + uintptr_t missMonitorAddr; /**< User-allocated miss monitor address */ + } externalHashParams; +#endif /* (DPAA_VERSION >= 11) */ } t_FmPcdHashTableParams; /**************************************************************************//** diff --git a/sys/contrib/ncsw/inc/Peripherals/qm_ext.h b/sys/contrib/ncsw/inc/Peripherals/qm_ext.h index 6a4bc9fcfe6f..9c20ee48d7c3 100644 --- a/sys/contrib/ncsw/inc/Peripherals/qm_ext.h +++ b/sys/contrib/ncsw/inc/Peripherals/qm_ext.h @@ -1,6 +1,6 @@ /****************************************************************************** - © 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc. + � 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc. All rights reserved. This is proprietary source code of Freescale Semiconductor Inc., @@ -258,6 +258,13 @@ typedef struct { NOTE: this parameter relevant only when working with multiple partitions. */ uint16_t partNumOfCgs; /**< Number of cgr's dedicated to this partition. NOTE: this parameter relevant only when working with multiple partitions. */ + void *p_FqdBase; /**< Pre-allocated FQD memory (virtual), or NULL for internal allocation. + Must be physically contiguous, naturally aligned, and non-cacheable. + When set, totalNumOfFqids is computed from fqdSize / 64. */ + uint32_t fqdSize; /**< Size of pre-allocated FQD memory in bytes. */ + void *p_PfdrBase; /**< Pre-allocated PFDR memory (virtual), or NULL for internal allocation. + Must be physically contiguous, naturally aligned, and non-cacheable. */ + uint32_t pfdrSize; /**< Size of pre-allocated PFDR memory in bytes. */ } t_QmParam; @@ -791,6 +798,40 @@ typedef struct t_QmPortalFrameInfo { *//***************************************************************************/ t_Error QM_PORTAL_Poll(t_Handle h_QmPortal, e_QmPortalPollSource source); +#ifdef __aarch64__ +void QM_PORTAL_DqrrDiag(t_Handle h_QmPortal, uint8_t *sw_pi, uint8_t *sw_ci, + uint8_t *sw_fill, uint8_t *hw_pi, uint8_t *hw_ci); +void QM_PORTAL_IsrDiag(t_Handle h_QmPortal, uint32_t *isr, uint32_t *ier, + uint32_t *iir); + +void QM_ErrorDiag(t_Handle h_Qm, uint32_t *err_isr, uint32_t *ecsr, + uint32_t *ecir, uint32_t *pfdr_fpc, uint32_t *sfdr_in_use, + uint32_t *idle_stat, uint32_t *dcp0_cfg, uint32_t *dcp0_dlm_avg, + uint32_t *dcp_dd_ihrsr, uint32_t *dcp_dd_hasr); + +/**************************************************************************//** + @Function QM_PORTAL_Inhibit + + @Description Inhibit all interrupts for this portal (IIR=1). + Used by NAPI-style FILTER handler to mask interrupts + at the source before scheduling deferred processing. + + @Param[in] h_QmPortal - A handle to a QM-Portal module +*//***************************************************************************/ +void QM_PORTAL_Inhibit(t_Handle h_QmPortal); + +/**************************************************************************//** + @Function QM_PORTAL_Uninhibit + + @Description Clear all pending ISR status and uninhibit portal + interrupts (IIR=0). Called after draining DQRR/MR + to re-enable interrupt delivery. + + @Param[in] h_QmPortal - A handle to a QM-Portal module +*//***************************************************************************/ +void QM_PORTAL_Uninhibit(t_Handle h_QmPortal); +#endif + /**************************************************************************//** @Function QM_PORTAL_PollFrame @@ -1058,6 +1099,21 @@ uint32_t QM_FQR_GetCounter(t_Handle h_QmFqr, t_Handle h_QmPortal, uint32_t fqidO *//***************************************************************************/ t_Error QM_FQR_Enqueue(t_Handle h_QmFqr, t_Handle h_QmPortal, uint32_t fqidOffset, t_DpaaFD *p_Frame); +/**************************************************************************//** + @Function QM_PORTAL_EnqueueFqid + + @Description Enqueue a frame to a specific FQID without requiring an FQR + handle. Used for CEETM frame queues which are managed by + CEETM hardware rather than NCSW FQR. + + @Param[in] h_QmPortal - A handle to a QM Portal Module. + @Param[in] fqid - The target Frame Queue ID. + @Param[in] p_Frame - Pointer to the frame to enqueue. + + @Return E_OK on success; Error code otherwise. +*//***************************************************************************/ +t_Error QM_PORTAL_EnqueueFqid(t_Handle h_QmPortal, uint32_t fqid, t_DpaaFD *p_Frame); + /**************************************************************************//** @Function QM_FQR_PullFrame diff --git a/sys/contrib/ncsw/inc/core_ext.h b/sys/contrib/ncsw/inc/core_ext.h index ec89a6dde4cd..8b4779672cc9 100644 --- a/sys/contrib/ncsw/inc/core_ext.h +++ b/sys/contrib/ncsw/inc/core_ext.h @@ -43,7 +43,11 @@ #ifndef __CORE_EXT_H #define __CORE_EXT_H -#ifdef CONFIG_FMAN_ARM +#if defined(NCSW_ARM_CORE) +/* FreeBSD ARM64 path — CORE_CACHELINE_SIZE set in dflags_arm64.h */ +#include "arm_ext.h" +#elif defined(CONFIG_FMAN_ARM) +/* Linux ARM path */ #include "arm_ext.h" #include #else @@ -63,7 +67,7 @@ #error "Must define the core cache-line size!" #endif /* !CORE_CACHELINE_SIZE */ -#endif /* CONFIG_FMAN_ARM */ +#endif /* NCSW_ARM_CORE / CONFIG_FMAN_ARM */ /**************************************************************************//** diff --git a/sys/contrib/ncsw/inc/cores/arm_ext.h b/sys/contrib/ncsw/inc/cores/arm_ext.h index e63444a7d676..5ab333d1efdb 100644 --- a/sys/contrib/ncsw/inc/cores/arm_ext.h +++ b/sys/contrib/ncsw/inc/cores/arm_ext.h @@ -35,9 +35,6 @@ @File arm_ext.h @Description Core API for ARM cores - - These routines must be implemented by each specific PowerPC - core driver. *//***************************************************************************/ #ifndef __ARM_EXT_H #define __ARM_EXT_H @@ -52,4 +49,28 @@ static __inline__ void CORE_MemoryBarrier(void) mb(); } -#endif /* __PPC_EXT_H */ +#if defined(NCSW_ARM_CORE) && defined(NCSW_FREEBSD) +/* + * FreeBSD ARM64 core function bindings. + * Implementations are in core_arm64.c. + * + * Follow the same pattern as PPC: e500v2_ext.h defines + * CORE_GetId -> E500_GetId, etc. We map to ARM64_* equivalents. + */ + +uint32_t ARM64_GetId(void); +int ARM64_TestAndSet(volatile int *p); +void ARM64_InstructionSync(void); + +#define CORE_GetId ARM64_GetId +#define CORE_TestAndSet ARM64_TestAndSet +#define CORE_InstructionSync ARM64_InstructionSync + +/* Cache management — handled by the kernel on ARM64, no-ops here */ +#define CORE_DCacheEnable() do { } while (0) +#define CORE_ICacheEnable() do { } while (0) +#define CORE_DCacheDisable() do { } while (0) +#define CORE_ICacheDisable() do { } while (0) +#endif /* NCSW_ARM_CORE && NCSW_FREEBSD */ + +#endif /* __ARM_EXT_H */ diff --git a/sys/contrib/ncsw/inc/integrations/dpaa_integration_ext.h b/sys/contrib/ncsw/inc/integrations/dpaa_integration_ext.h index 27bb5d402378..7003e0279714 100644 --- a/sys/contrib/ncsw/inc/integrations/dpaa_integration_ext.h +++ b/sys/contrib/ncsw/inc/integrations/dpaa_integration_ext.h @@ -103,6 +103,45 @@ typedef enum e_QM_FQ_CHANNEL_SWPORTAL8, e_QM_FQ_CHANNEL_SWPORTAL9, +#ifdef __aarch64__ + /* QMan v3 (LS1046A): pool channels at 0x401-0x40F, FMan DC at 0x800+, + * CAAM at 0x840. SW portal channels (0-9) are unchanged. */ + e_QM_FQ_CHANNEL_POOL1 = 0x401, + e_QM_FQ_CHANNEL_POOL2, + e_QM_FQ_CHANNEL_POOL3, + e_QM_FQ_CHANNEL_POOL4, + e_QM_FQ_CHANNEL_POOL5, + e_QM_FQ_CHANNEL_POOL6, + e_QM_FQ_CHANNEL_POOL7, + e_QM_FQ_CHANNEL_POOL8, + e_QM_FQ_CHANNEL_POOL9, + e_QM_FQ_CHANNEL_POOL10, + e_QM_FQ_CHANNEL_POOL11, + e_QM_FQ_CHANNEL_POOL12, + e_QM_FQ_CHANNEL_POOL13, + e_QM_FQ_CHANNEL_POOL14, + e_QM_FQ_CHANNEL_POOL15, + + e_QM_FQ_CHANNEL_FMAN0_SP0 = 0x800, + e_QM_FQ_CHANNEL_FMAN0_SP1, + e_QM_FQ_CHANNEL_FMAN0_SP2, + e_QM_FQ_CHANNEL_FMAN0_SP3, + e_QM_FQ_CHANNEL_FMAN0_SP4, + e_QM_FQ_CHANNEL_FMAN0_SP5, + e_QM_FQ_CHANNEL_FMAN0_SP6, + e_QM_FQ_CHANNEL_FMAN0_SP7, + e_QM_FQ_CHANNEL_FMAN0_SP8, + e_QM_FQ_CHANNEL_FMAN0_SP9, + e_QM_FQ_CHANNEL_FMAN0_SP10, + e_QM_FQ_CHANNEL_FMAN0_SP11, + e_QM_FQ_CHANNEL_FMAN0_SP12, + e_QM_FQ_CHANNEL_FMAN0_SP13, + e_QM_FQ_CHANNEL_FMAN0_SP14, + e_QM_FQ_CHANNEL_FMAN0_SP15, + + e_QM_FQ_CHANNEL_CAAM = 0x840 +#else + /* QMan v1/v2 (P5020): pool channels at 0x21-0x2F, FMan DC at 0x40+ */ e_QM_FQ_CHANNEL_POOL1 = 0x21, /**< Pool channels that can be serviced by any of the software portals */ e_QM_FQ_CHANNEL_POOL2, e_QM_FQ_CHANNEL_POOL3, @@ -144,6 +183,7 @@ typedef enum connected to PME */ e_QM_FQ_CHANNEL_RAID = 0xC0 /**< Dedicated channel serviced by Direct Connect Portal 4: connected to RAID */ +#endif } e_QmFQChannel; /***************************************************************************** @@ -156,11 +196,31 @@ typedef enum ******************************************************************************/ #define INTG_MAX_NUM_OF_FM 1 +/* + * ---------- Architecture-dependent integration parameters ---------- + * + * ARM64 (LS1046A / FManv3, DPAA 1.1) + * Values from NXP Linux SDK LS1043 dpaa_integration_ext.h + * PowerPC (P5020 / FManv2, DPAA 1.0) + * Original FreeBSD values + */ +#ifdef __aarch64__ +#define DPAA_VERSION 11 +#else +#define DPAA_VERSION 10 +#endif + /* Ports defines */ +#ifdef __aarch64__ +#define FM_MAX_NUM_OF_1G_MACS 6 +#define FM_MAX_NUM_OF_10G_MACS 2 +#define FM_MAX_NUM_OF_OH_PORTS 6 +#else #define FM_MAX_NUM_OF_1G_MACS 5 #define FM_MAX_NUM_OF_10G_MACS 1 -#define FM_MAX_NUM_OF_MACS (FM_MAX_NUM_OF_1G_MACS + FM_MAX_NUM_OF_10G_MACS) #define FM_MAX_NUM_OF_OH_PORTS 7 +#endif +#define FM_MAX_NUM_OF_MACS (FM_MAX_NUM_OF_1G_MACS + FM_MAX_NUM_OF_10G_MACS) #define FM_MAX_NUM_OF_1G_RX_PORTS FM_MAX_NUM_OF_1G_MACS #define FM_MAX_NUM_OF_10G_RX_PORTS FM_MAX_NUM_OF_10G_MACS @@ -170,46 +230,85 @@ typedef enum #define FM_MAX_NUM_OF_10G_TX_PORTS FM_MAX_NUM_OF_10G_MACS #define FM_MAX_NUM_OF_TX_PORTS (FM_MAX_NUM_OF_10G_TX_PORTS + FM_MAX_NUM_OF_1G_TX_PORTS) +#ifdef __aarch64__ +#define FM_PORT_MAX_NUM_OF_EXT_POOLS 4 /**< Number of external BM pools per Rx port */ +#define FM_MAX_NUM_OF_SUB_PORTALS 16 +#else #define FM_PORT_MAX_NUM_OF_EXT_POOLS 8 /**< Number of external BM pools per Rx port */ -#define FM_PORT_NUM_OF_CONGESTION_GRPS 256 /**< Total number of congestion groups in QM */ #define FM_MAX_NUM_OF_SUB_PORTALS 12 +#endif +#define FM_PORT_NUM_OF_CONGESTION_GRPS 256 /**< Total number of congestion groups in QM */ #define FM_PORT_MAX_NUM_OF_OBSERVED_EXT_POOLS 0 +#if (DPAA_VERSION >= 11) +#define FM_VSP_MAX_NUM_OF_ENTRIES 64 +#define FM_MAX_NUM_OF_PFC_PRIORITIES 8 +#endif + /* RAMs defines */ -#define FM_MURAM_SIZE (160 * KILOBYTE) +#ifdef __aarch64__ +#define FM_MURAM_SIZE (384 * KILOBYTE) /**< LS1046A FManv3: 384KB MURAM */ +#else +#define FM_MURAM_SIZE (160 * KILOBYTE) /**< P5020 FManv2: 160KB MURAM */ +#endif #define FM_IRAM_SIZE(a,b) ( 64 * KILOBYTE) /* PCD defines */ #define FM_PCD_PLCR_NUM_ENTRIES 256 /**< Total number of policer profiles */ #define FM_PCD_KG_NUM_OF_SCHEMES 32 /**< Total number of KG schemes */ #define FM_PCD_MAX_NUM_OF_CLS_PLANS 256 /**< Number of classification plan entries. */ +#define FM_PCD_SW_PRS_SIZE 0x00000800 +#ifdef __aarch64__ +#define FM_PCD_PRS_SW_PATCHES_SIZE 0x00000600 +#else +#define FM_PCD_PRS_SW_PATCHES_SIZE 0x00000200 +#endif /* RTC defines */ #define FM_RTC_NUM_OF_ALARMS 2 /**< RTC number of alarms */ +#ifdef __aarch64__ +#define FM_RTC_NUM_OF_PERIODIC_PULSES 3 /**< RTC number of periodic pulses */ +#else #define FM_RTC_NUM_OF_PERIODIC_PULSES 2 /**< RTC number of periodic pulses */ +#endif #define FM_RTC_NUM_OF_EXT_TRIGGERS 2 /**< RTC number of external triggers */ /* QMI defines */ #define QMI_MAX_NUM_OF_TNUMS 64 +#ifdef __aarch64__ +#define QMI_DEF_TNUMS_THRESH 32 +#else #define MAX_QMI_DEQ_SUBPORTAL 12 #define QMI_DEF_TNUMS_THRESH 48 +#endif /* FPM defines */ #define FM_NUM_OF_FMAN_CTRL_EVENT_REGS 4 /* DMA defines */ +#ifdef __aarch64__ +#define DMA_THRESH_MAX_COMMQ 83 +#else #define DMA_THRESH_MAX_COMMQ 31 +#endif #define DMA_THRESH_MAX_BUF 127 /* BMI defines */ #define BMI_MAX_NUM_OF_TASKS 128 +#ifdef __aarch64__ +#define BMI_MAX_NUM_OF_DMAS 84 +#define FM_NUM_OF_CTRL 4 +#else #define BMI_MAX_NUM_OF_DMAS 32 +#define FM_NUM_OF_CTRL 2 +#endif #define BMI_MAX_FIFO_SIZE (FM_MURAM_SIZE) #define PORT_MAX_WEIGHT 16 #define FM_CHECK_PORT_RESTRICTIONS(__validPorts, __newPortIndx) TRUE +#ifndef __aarch64__ /* P5020 unique features */ #define FM_QMI_DEQ_OPTIONS_SUPPORT #define FM_NO_DISPATCH_RAM_ECC @@ -250,8 +349,6 @@ typedef enum #define FM_IM_LARGE_MRBLR_ERRATA_FMAN15 /* Implemented by ucode */ #define FM_BMI_TO_RISC_ENQ_ERRATA_FMANc /* No implementation, Out of LLD scope */ #define FM_INVALID_SWPRS_DATA_ERRATA_FMANd -//#define FM_PRS_MPLS_SSA_ERRATA_FMANj /* No implementation, No patch yet */ -//#define FM_PRS_INITIAL_PLANID_ERRATA_FMANk /* No implementation, No patch yet */ #define FM_NO_COPY_CTXA_CTXB_ERRATA_FMAN_SW001 @@ -260,16 +357,35 @@ typedef enum /* P2041 */ #define FM_BAD_VLAN_DETECT_ERRATA_10GMAC_A010 -/* Common to all */ +/* Common to P5020 */ #define FM_RX_PREAM_4_ERRATA_DTSEC_A001 FM_NO_RX_PREAM_ERRATA_DTSECx1 #define FM_UCODE_NOT_RESET_ERRATA_BUGZILLA6173 #define FM_MAGIC_PACKET_UNRECOGNIZED_ERRATA_DTSEC2 /* No implementation, Out of LLD scope */ #define FM_PRS_MEM_ERRATA_FMAN_SW003 -#define FM_LEN_CHECK_ERRATA_FMAN_SW002 -#define DPAA_VERSION 10 -#define FM_PCD_SW_PRS_SIZE 0x00000800 -#define FM_PCD_PRS_SW_PATCHES_SIZE 0x00000200 -#define FM_NUM_OF_CTRL 2 +#else /* __aarch64__ — LS1046A / FManv3 errata and features */ + +#define FM_OP_OPEN_DMA_MIN_LIMIT +#define FM_NO_RESTRICT_ON_ACCESS_RSRC +#define FM_NO_OP_OBSERVED_POOLS +#define FM_FRAME_END_PARAMS_FOR_OP +#define FM_DEQ_PIPELINE_PARAMS_FOR_OP +#define FM_QMI_NO_SINGLE_ECC_EXCEPTION +#define FM_NO_GUARANTEED_RESET_VALUES + +/* LS1046A / FManv3 errata */ +#define FM_HEAVY_TRAFFIC_HANG_ERRATA_FMAN_A005669 +#define FM_WRONG_RESET_VALUES_ERRATA_FMAN_A005127 +#define FM_RX_FIFO_CORRUPT_ERRATA_10GMAC_A006320 +#define FM_OP_NO_VSP_NO_RELEASE_ERRATA_FMAN_A006675 +#define FM_HEAVY_TRAFFIC_SEQUENCER_HANG_ERRATA_FMAN_A006981 +#define FM_BCB_ERRATA_BMI_SW001 +#define FM_AID_MODE_NO_TNUM_SW005 +#define FM_ERROR_VSP_NO_MATCH_SW006 + +#endif /* __aarch64__ */ + +/* Common to all platforms */ +#define FM_LEN_CHECK_ERRATA_FMAN_SW002 #endif /* __DPAA_INTEGRATION_EXT_H */ diff --git a/sys/contrib/ncsw/inc/integrations/part_ext.h b/sys/contrib/ncsw/inc/integrations/part_ext.h index 0fa702993892..614fc499a374 100644 --- a/sys/contrib/ncsw/inc/integrations/part_ext.h +++ b/sys/contrib/ncsw/inc/integrations/part_ext.h @@ -44,8 +44,14 @@ #include "enet_ext.h" #include "dpaa_integration_ext.h" +#if defined(NCSW_ARM_CORE) +/* ARM64 — LS1046A has 4x Cortex-A72; no PPC core type */ +#define INTG_MAX_NUM_OF_CORES 4 +#else +/* PowerPC — P5020/P3041/P5040 */ #define CORE_E500MC #define INTG_MAX_NUM_OF_CORES 24 +#endif /**************************************************************************//* @Description Part data structure - must be contained in any integration diff --git a/sys/contrib/ncsw/inc/ioctls/fm_ioctls.h b/sys/contrib/ncsw/inc/ioctls/fm_ioctls.h new file mode 100644 index 000000000000..e4838bd36562 --- /dev/null +++ b/sys/contrib/ncsw/inc/ioctls/fm_ioctls.h @@ -0,0 +1,645 @@ +/* Copyright (c) 2008-2012 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/**************************************************************************//** + @File fm_ioctls.h + + @Description FM Char device ioctls +*//***************************************************************************/ +#ifndef __FM_IOCTLS_H +#define __FM_IOCTLS_H + + +/**************************************************************************//** + @Group lnx_ioctl_FM_grp Frame Manager Linux IOCTL API + + @Description FM Linux ioctls definitions and enums + + @{ +*//***************************************************************************/ + +/**************************************************************************//** + @Collection FM IOCTL device ('/dev') definitions +*//***************************************************************************/ +#define DEV_FM_NAME "fm" /**< Name of the FM chardev */ + +#define DEV_FM_MINOR_BASE 0 +#define DEV_FM_PCD_MINOR_BASE (DEV_FM_MINOR_BASE + 1) /*/dev/fmx-pcd */ +#define DEV_FM_OH_PORTS_MINOR_BASE (DEV_FM_PCD_MINOR_BASE + 1) /*/dev/fmx-port-ohy */ +#define DEV_FM_RX_PORTS_MINOR_BASE (DEV_FM_OH_PORTS_MINOR_BASE + FM_MAX_NUM_OF_OH_PORTS) /*/dev/fmx-port-rxy */ +#define DEV_FM_TX_PORTS_MINOR_BASE (DEV_FM_RX_PORTS_MINOR_BASE + FM_MAX_NUM_OF_RX_PORTS) /*/dev/fmx-port-txy */ +#define DEV_FM_MAX_MINORS (DEV_FM_TX_PORTS_MINOR_BASE + FM_MAX_NUM_OF_TX_PORTS) + +#define FM_IOC_NUM(n) (n) +#define FM_PCD_IOC_NUM(n) (n+20) +#define FM_PORT_IOC_NUM(n) (n+70) +/* @} */ + +#define IOC_FM_MAX_NUM_OF_PORTS 64 + + +/**************************************************************************//** + @Description Enum for defining port types + (must match enum e_FmPortType defined in fm_ext.h) +*//***************************************************************************/ +typedef enum ioc_fm_port_type { + e_IOC_FM_PORT_TYPE_OH_OFFLINE_PARSING = 0, /**< Offline parsing port */ + e_IOC_FM_PORT_TYPE_RX, /**< 1G Rx port */ + e_IOC_FM_PORT_TYPE_RX_10G, /**< 10G Rx port */ + e_IOC_FM_PORT_TYPE_TX, /**< 1G Tx port */ + e_IOC_FM_PORT_TYPE_TX_10G, /**< 10G Tx port */ + e_IOC_FM_PORT_TYPE_DUMMY +} ioc_fm_port_type; + + +/**************************************************************************//** + @Group lnx_ioctl_FM_lib_grp FM library + + @Description FM API functions, definitions and enums + The FM module is the main driver module and is a mandatory module + for FM driver users. Before any further module initialization, + this module must be initialized. + The FM is a "single-tone" module. It is responsible of the common + HW modules: FPM, DMA, common QMI, common BMI initializations and + run-time control routines. This module must be initialized always + when working with any of the FM modules. + NOTE - We assumes that the FML will be initialize only by core No. 0! + + @{ +*//***************************************************************************/ + +/**************************************************************************//** + @Description FM Exceptions +*//***************************************************************************/ +typedef enum ioc_fm_exceptions { + e_IOC_FM_EX_DMA_BUS_ERROR, /**< DMA bus error. */ + e_IOC_EX_DMA_READ_ECC, /**< Read Buffer ECC error (Valid for FM rev < 6)*/ + e_IOC_EX_DMA_SYSTEM_WRITE_ECC, /**< Write Buffer ECC error on system side (Valid for FM rev < 6)*/ + e_IOC_EX_DMA_FM_WRITE_ECC, /**< Write Buffer ECC error on FM side (Valid for FM rev < 6)*/ + e_IOC_EX_DMA_SINGLE_PORT_ECC, /**< Single Port ECC error on FM side (Valid for FM rev > 6)*/ + e_IOC_EX_FPM_STALL_ON_TASKS, /**< Stall of tasks on FPM */ + e_IOC_EX_FPM_SINGLE_ECC, /**< Single ECC on FPM. */ + e_IOC_EX_FPM_DOUBLE_ECC, /**< Double ECC error on FPM ram access */ + e_IOC_EX_QMI_SINGLE_ECC, /**< Single ECC on QMI. */ + e_IOC_EX_QMI_DOUBLE_ECC, /**< Double bit ECC occurred on QMI */ + e_IOC_EX_QMI_DEQ_FROM_UNKNOWN_PORTID,/**< Dequeue from unknown port id */ + e_IOC_EX_BMI_LIST_RAM_ECC, /**< Linked List RAM ECC error */ + e_IOC_EX_BMI_STORAGE_PROFILE_ECC, /**< Storage Profile ECC Error */ + e_IOC_EX_BMI_STATISTICS_RAM_ECC, /**< Statistics Count RAM ECC Error Enable */ + e_IOC_EX_BMI_DISPATCH_RAM_ECC, /**< Dispatch RAM ECC Error Enable */ + e_IOC_EX_IRAM_ECC, /**< Double bit ECC occurred on IRAM*/ + e_IOC_EX_MURAM_ECC /**< Double bit ECC occurred on MURAM*/ +} ioc_fm_exceptions; + +/**************************************************************************//** + @Group lnx_ioctl_FM_runtime_control_grp FM Runtime Control Unit + + @Description FM Runtime control unit API functions, definitions and enums. + The FM driver provides a set of control routines for each module. + These routines may only be called after the module was fully + initialized (both configuration and initialization routines were + called). They are typically used to get information from hardware + (status, counters/statistics, revision etc.), to modify a current + state or to force/enable a required action. Run-time control may + be called whenever necessary and as many times as needed. + @{ +*//***************************************************************************/ + +/**************************************************************************//** + @Collection General FM defines. + *//***************************************************************************/ +#define IOC_FM_MAX_NUM_OF_VALID_PORTS (FM_MAX_NUM_OF_OH_PORTS + \ + FM_MAX_NUM_OF_1G_RX_PORTS + \ + FM_MAX_NUM_OF_10G_RX_PORTS + \ + FM_MAX_NUM_OF_1G_TX_PORTS + \ + FM_MAX_NUM_OF_10G_TX_PORTS) +/* @} */ + +/**************************************************************************//** + @Description Structure for Port bandwidth requirement. Port is identified + by type and relative id. + (must be identical to t_FmPortBandwidth defined in fm_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_port_bandwidth_t { + ioc_fm_port_type type; /**< FM port type */ + uint8_t relative_port_id; /**< Type relative port id */ + uint8_t bandwidth; /**< bandwidth - (in term of percents) */ +} ioc_fm_port_bandwidth_t; + +/**************************************************************************//** + @Description A Structure containing an array of Port bandwidth requirements. + The user should state the ports requiring bandwidth in terms of + percentage - i.e. all port's bandwidths in the array must add + up to 100. + (must be identical to t_FmPortsBandwidthParams defined in fm_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_port_bandwidth_params { + uint8_t num_of_ports; + /**< num of ports listed in the array below */ + ioc_fm_port_bandwidth_t ports_bandwidths[IOC_FM_MAX_NUM_OF_VALID_PORTS]; + /**< for each port, it's bandwidth (all port's + bandwidths must add up to 100.*/ +} ioc_fm_port_bandwidth_params; + +/**************************************************************************//** + @Description enum for defining FM counters +*//***************************************************************************/ +typedef enum ioc_fm_counters { + e_IOC_FM_COUNTERS_ENQ_TOTAL_FRAME, /**< QMI total enqueued frames counter */ + e_IOC_FM_COUNTERS_DEQ_TOTAL_FRAME, /**< QMI total dequeued frames counter */ + e_IOC_FM_COUNTERS_DEQ_0, /**< QMI 0 frames from QMan counter */ + e_IOC_FM_COUNTERS_DEQ_1, /**< QMI 1 frames from QMan counter */ + e_IOC_FM_COUNTERS_DEQ_2, /**< QMI 2 frames from QMan counter */ + e_IOC_FM_COUNTERS_DEQ_3, /**< QMI 3 frames from QMan counter */ + e_IOC_FM_COUNTERS_DEQ_FROM_DEFAULT, /**< QMI dequeue from default queue counter */ + e_IOC_FM_COUNTERS_DEQ_FROM_CONTEXT, /**< QMI dequeue from FQ context counter */ + e_IOC_FM_COUNTERS_DEQ_FROM_FD, /**< QMI dequeue from FD command field counter */ + e_IOC_FM_COUNTERS_DEQ_CONFIRM, /**< QMI dequeue confirm counter */ +} ioc_fm_counters; + +typedef struct ioc_fm_obj_t { + void *obj; +} ioc_fm_obj_t; + +/**************************************************************************//** + @Description A structure for returning revision information + (must match struct t_FmRevisionInfo declared in fm_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_revision_info_t { + uint8_t major; /**< Major revision */ + uint8_t minor; /**< Minor revision */ +} ioc_fm_revision_info_t; + +/**************************************************************************//** + @Description A structure for FM counters +*//***************************************************************************/ +typedef struct ioc_fm_counters_params_t { + ioc_fm_counters cnt; /**< The requested counter */ + uint32_t val; /**< The requested value to get/set from/into the counter */ +} ioc_fm_counters_params_t; + +typedef union ioc_fm_api_version_t { + struct { + uint8_t major; + uint8_t minor; + uint8_t respin; + uint8_t reserved; + } version; + uint32_t ver; +} ioc_fm_api_version_t; + +/**************************************************************************//** + @Description A structure of information about each of the external + buffer pools used by a port or storage-profile. + (must be identical to t_FmExtPoolParams defined in fm_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_ext_pool_params { + uint8_t id; /**< External buffer pool id */ + uint16_t size; /**< External buffer pool buffer size */ +} ioc_fm_ext_pool_params; + +/**************************************************************************//** + @Description A structure for informing the driver about the external + buffer pools allocated in the BM and used by a port or a + storage-profile. + (must be identical to t_FmExtPools defined in fm_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_ext_pools { + uint8_t num_of_pools_used; /**< Number of pools use by this port */ + ioc_fm_ext_pool_params ext_buf_pool[FM_PORT_MAX_NUM_OF_EXT_POOLS]; + /**< Parameters for each port */ +} ioc_fm_ext_pools; + +typedef struct ioc_fm_vsp_params_t { + void *p_fm; /**< A handle to the FM object this VSP related to */ + ioc_fm_ext_pools ext_buf_pools; /**< Which external buffer pools are used + (up to FM_PORT_MAX_NUM_OF_EXT_POOLS), and their sizes. + parameter associated with Rx / OP port */ + uint16_t liodn_offset; /**< VSP's LIODN offset */ + struct { + ioc_fm_port_type port_type; /**< Port type */ + uint8_t port_id; /**< Port Id - relative to type */ + } port_params; + uint8_t relative_profile_id; /**< VSP Id - relative to VSP's range + defined in relevant FM object */ + void *id; /**< return value */ +} ioc_fm_vsp_params_t; + +/**************************************************************************//** + @Description A structure for defining BM pool depletion criteria +*//***************************************************************************/ +typedef struct ioc_fm_buf_pool_depletion_t { + bool pools_grp_mode_enable; /**< select mode in which pause frames will be sent after + a number of pools (all together!) are depleted */ + uint8_t num_of_pools; /**< the number of depleted pools that will invoke + pause frames transmission. */ + bool pools_to_consider[BM_MAX_NUM_OF_POOLS]; + /**< For each pool, TRUE if it should be considered for + depletion (Note - this pool must be used by this port!). */ + bool single_pool_mode_enable; /**< select mode in which pause frames will be sent after + a single-pool is depleted; */ + bool pools_to_consider_for_single_mode[BM_MAX_NUM_OF_POOLS]; + /**< For each pool, TRUE if it should be considered for + depletion (Note - this pool must be used by this port!) */ + bool pfc_priorities_en[FM_MAX_NUM_OF_PFC_PRIORITIES]; + /**< This field is used by the MAC as the Priority Enable Vector in the PFC frame + which is transmitted */ +} ioc_fm_buf_pool_depletion_t; + +typedef struct ioc_fm_buf_pool_depletion_params_t { + void *p_fm_vsp; + ioc_fm_buf_pool_depletion_t fm_buf_pool_depletion; +} ioc_fm_buf_pool_depletion_params_t; + +typedef struct ioc_fm_buffer_prefix_content_t { + uint16_t priv_data_size; /**< Number of bytes to be left at the beginning + of the external buffer; Note that the private-area will + start from the base of the buffer address. */ + bool pass_prs_result; /**< TRUE to pass the parse result to/from the FM; + User may use FM_PORT_GetBufferPrsResult() in order to + get the parser-result from a buffer. */ + bool pass_time_stamp; /**< TRUE to pass the timeStamp to/from the FM + User may use FM_PORT_GetBufferTimeStamp() in order to + get the parser-result from a buffer. */ + bool pass_hash_result; /**< TRUE to pass the KG hash result to/from the FM + User may use FM_PORT_GetBufferHashResult() in order to + get the parser-result from a buffer. */ + bool pass_all_other_pcd_info; /**< Add all other Internal-Context information: + AD, hash-result, key, etc. */ + uint16_t data_align; /**< 0 to use driver's default alignment [64], + other value for selecting a data alignment (must be a power of 2); + if write optimization is used, must be >= 16. */ + uint8_t manip_extra_space; /**< Maximum extra size needed (insertion-size minus removal-size); + Note that this field impacts the size of the buffer-prefix + (i.e. it pushes the data offset); */ +} ioc_fm_buffer_prefix_content_t; + +typedef struct ioc_fm_buffer_prefix_content_params_t { + void *p_fm_vsp; + ioc_fm_buffer_prefix_content_t fm_buffer_prefix_content; +} ioc_fm_buffer_prefix_content_params_t; + +typedef struct ioc_fm_vsp_config_no_sg_params_t { + void *p_fm_vsp; + bool no_sg; +} ioc_fm_vsp_config_no_sg_params_t; + +typedef struct ioc_fm_vsp_prs_result_params_t { + void *p_fm_vsp; + void *p_data; +} ioc_fm_vsp_prs_result_params_t; + +typedef struct fm_ctrl_mon_t { + uint8_t percent_cnt[2]; +} fm_ctrl_mon_t; + +typedef struct ioc_fm_ctrl_mon_counters_params_t { + uint8_t fm_ctrl_index; + fm_ctrl_mon_t *p_mon; +} ioc_fm_ctrl_mon_counters_params_t; + +/**************************************************************************//** + @Function FM_IOC_SET_PORTS_BANDWIDTH + + @Description Sets relative weights between ports when accessing common resources. + + @Param[in] ioc_fm_port_bandwidth_params Port bandwidth percentages, + their sum must equal 100. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#define FM_IOC_SET_PORTS_BANDWIDTH _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(2), ioc_fm_port_bandwidth_params) + +/**************************************************************************//** + @Function FM_IOC_GET_REVISION + + @Description Returns the FM revision + + @Param[out] ioc_fm_revision_info_t A structure of revision information parameters. + + @Return None. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#define FM_IOC_GET_REVISION _IOR(FM_IOC_TYPE_BASE, FM_IOC_NUM(3), ioc_fm_revision_info_t) + +/**************************************************************************//** + @Function FM_IOC_GET_COUNTER + + @Description Reads one of the FM counters. + + @Param[in,out] ioc_fm_counters_params_t The requested counter parameters. + + @Return Counter's current value. + + @Cautions Allowed only following FM_Init(). + Note that it is user's responsibilty to call this routine only + for enabled counters, and there will be no indication if a + disabled counter is accessed. +*//***************************************************************************/ +#define FM_IOC_GET_COUNTER _IOWR(FM_IOC_TYPE_BASE, FM_IOC_NUM(4), ioc_fm_counters_params_t) + +/**************************************************************************//** + @Function FM_IOC_SET_COUNTER + + @Description Sets a value to an enabled counter. Use "0" to reset the counter. + + @Param[in] ioc_fm_counters_params_t The requested counter parameters. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#define FM_IOC_SET_COUNTER _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(5), ioc_fm_counters_params_t) + +/**************************************************************************//** + @Function FM_IOC_FORCE_INTR + + @Description Causes an interrupt event on the requested source. + + @Param[in] ioc_fm_exceptions An exception to be forced. + + @Return E_OK on success; Error code if the exception is not enabled, + or is not able to create interrupt. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#define FM_IOC_FORCE_INTR _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(6), ioc_fm_exceptions) + +/**************************************************************************//** + @Function FM_IOC_GET_API_VERSION + + @Description Reads the FMD IOCTL API version. + + @Param[in,out] ioc_fm_api_version_t The requested counter parameters. + + @Return Version's value. +*//***************************************************************************/ +#define FM_IOC_GET_API_VERSION _IOR(FM_IOC_TYPE_BASE, FM_IOC_NUM(7), ioc_fm_api_version_t) + +/**************************************************************************//** + @Function FM_VSP_Config + + @Description Creates descriptor for the FM VSP module. + + The routine returns a handle (descriptor) to the FM VSP object. + This descriptor must be passed as first parameter to all other + FM VSP function calls. + + No actual initialization or configuration of FM hardware is + done by this routine. + +@Param[in] p_FmVspParams Pointer to data structure of parameters + + @Retval Handle to FM VSP object, or NULL for Failure. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_IOC_VSP_CONFIG_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_IOC_NUM(8), ioc_compat_fm_vsp_params_t) +#endif +#define FM_IOC_VSP_CONFIG _IOWR(FM_IOC_TYPE_BASE, FM_IOC_NUM(8), ioc_fm_vsp_params_t) + +/**************************************************************************//** + @Function FM_VSP_Init + + @Description Initializes the FM VSP module + + @Param[in] h_FmVsp - FM VSP module descriptor + + @Return E_OK on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_IOC_VSP_INIT_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(9), ioc_compat_fm_obj_t) +#endif +#define FM_IOC_VSP_INIT _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(9), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_VSP_Free + + @Description Frees all resources that were assigned to FM VSP module. + + Calling this routine invalidates the descriptor. + + @Param[in] h_FmVsp - FM VSP module descriptor + + @Return E_OK on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_IOC_VSP_FREE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(10), ioc_compat_fm_obj_t) +#endif +#define FM_IOC_VSP_FREE _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(10), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_VSP_ConfigPoolDepletion + + @Description Calling this routine enables pause frame generation depending on the + depletion status of BM pools. It also defines the conditions to activate + this functionality. By default, this functionality is disabled. + + @Param[in] ioc_fm_buf_pool_depletion_params_t A structure holding the required parameters. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_VSP_Config() and before FM_VSP_Init(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_IOC_VSP_CONFIG_POOL_DEPLETION_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(11), ioc_compat_fm_buf_pool_depletion_params_t) +#endif +#define FM_IOC_VSP_CONFIG_POOL_DEPLETION _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(11), ioc_fm_buf_pool_depletion_params_t) + +/**************************************************************************//** + @Function FM_VSP_ConfigBufferPrefixContent + + @Description Defines the structure, size and content of the application buffer. + + The prefix will + In VSPs defined for Tx ports, if 'passPrsResult', the application + should set a value to their offsets in the prefix of + the FM will save the first 'privDataSize', than, + depending on 'passPrsResult' and 'passTimeStamp', copy parse result + and timeStamp, and the packet itself (in this order), to the + application buffer, and to offset. + + Calling this routine changes the buffer margins definitions + in the internal driver data base from its default + configuration: Data size: [DEFAULT_FM_SP_bufferPrefixContent_privDataSize] + Pass Parser result: [DEFAULT_FM_SP_bufferPrefixContent_passPrsResult]. + Pass timestamp: [DEFAULT_FM_SP_bufferPrefixContent_passTimeStamp]. + + @Param[in] ioc_fm_buffer_prefix_content_params_t A structure holding the required parameters. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_VSP_Config() and before FM_VSP_Init(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_IOC_VSP_CONFIG_BUFFER_PREFIX_CONTENT_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(12), ioc_compat_fm_buffer_prefix_content_params_t) +#endif +#define FM_IOC_VSP_CONFIG_BUFFER_PREFIX_CONTENT _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(12), ioc_fm_buffer_prefix_content_params_t) + +/**************************************************************************//** + @Function FM_VSP_ConfigNoScatherGather + + @Description Calling this routine changes the possibility to receive S/G frame + in the internal driver data base + from its default configuration: optimize = [DEFAULT_FM_SP_noScatherGather] + + @Param[in] ioc_fm_vsp_config_no_sg_params_t A structure holding the required parameters. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_VSP_Config() and before FM_VSP_Init(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_IOC_VSP_CONFIG_NO_SG_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(13), ioc_compat_fm_vsp_config_no_sg_params_t) +#endif +#define FM_IOC_VSP_CONFIG_NO_SG _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(13), ioc_fm_vsp_config_no_sg_params_t) + +/**************************************************************************//** + @Function FM_VSP_GetBufferPrsResult + + @Description Returns the pointer to the parse result in the data buffer. + In Rx ports this is relevant after reception, if parse + result is configured to be part of the data passed to the + application. For non Rx ports it may be used to get the pointer + of the area in the buffer where parse result should be + initialized - if so configured. + See FM_VSP_ConfigBufferPrefixContent for data buffer prefix + configuration. + + @Param[in] ioc_fm_vsp_prs_result_params_t A structure holding the required parameters. + + @Return Parse result pointer on success, NULL if parse result was not + configured for this port. + + @Cautions Allowed only following FM_VSP_Init(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_IOC_VSP_GET_BUFFER_PRS_RESULT_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_IOC_NUM(14), ioc_compat_fm_vsp_prs_result_params_t) +#endif +#define FM_IOC_VSP_GET_BUFFER_PRS_RESULT _IOWR(FM_IOC_TYPE_BASE, FM_IOC_NUM(14), ioc_fm_vsp_prs_result_params_t) + +/**************************************************************************//** + @Function FM_CtrlMonStart + + @Description Start monitoring utilization of all available FM controllers. + + In order to obtain FM controllers utilization the following sequence + should be used: + -# FM_CtrlMonStart() + -# FM_CtrlMonStop() + -# FM_CtrlMonGetCounters() - issued for each FM controller + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#define FM_IOC_CTRL_MON_START _IO(FM_IOC_TYPE_BASE, FM_IOC_NUM(15)) + + +/**************************************************************************//** + @Function FM_CtrlMonStop + + @Description Stop monitoring utilization of all available FM controllers. + + In order to obtain FM controllers utilization the following sequence + should be used: + -# FM_CtrlMonStart() + -# FM_CtrlMonStop() + -# FM_CtrlMonGetCounters() - issued for each FM controller + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#define FM_IOC_CTRL_MON_STOP _IO(FM_IOC_TYPE_BASE, FM_IOC_NUM(16)) + +/**************************************************************************//** + @Function FM_CtrlMonGetCounters + + @Description Obtain FM controller utilization parameters. + + In order to obtain FM controllers utilization the following sequence + should be used: + -# FM_CtrlMonStart() + -# FM_CtrlMonStop() + -# FM_CtrlMonGetCounters() - issued for each FM controller + + @Param[in] ioc_fm_ctrl_mon_counters_params_t A structure holding the required parameters. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_IOC_CTRL_MON_GET_COUNTERS_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(17), ioc_compat_fm_ctrl_mon_counters_params_t) +#endif +#define FM_IOC_CTRL_MON_GET_COUNTERS _IOW(FM_IOC_TYPE_BASE, FM_IOC_NUM(17), ioc_fm_ctrl_mon_counters_params_t) + +/**************************************************************************//** + @Function FM_ReadTimeStamp + + @Description Reads the FMan engine's timestamp. + + @Param[out] uint32_t The indicated engine's timestamp + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#define FM_IOC_READ_TIMESTAMP _IOWR(FM_IOC_TYPE_BASE, FM_IOC_NUM(18), uint32_t) + +/**************************************************************************//** + @Function FM_GetTimeStampIncrementPerUsec + + @Description Provides the value of the FMan engine's timestamp increment + per microsecond. + + @Param[out] uint32_t The value the timestamp is + incremented with each microsecond + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_Init(). +*//***************************************************************************/ +#define FM_IOC_GET_TIMESTAMP_INCREMENT _IOWR(FM_IOC_TYPE_BASE, FM_IOC_NUM(19), uint32_t) + +/** @} */ /* end of lnx_ioctl_FM_runtime_control_grp group */ +/** @} */ /* end of lnx_ioctl_FM_lib_grp group */ +/** @} */ /* end of lnx_ioctl_FM_grp */ + +#define FMD_API_VERSION_MAJOR 21 +#define FMD_API_VERSION_MINOR 1 +#define FMD_API_VERSION_RESPIN 0 + +#endif /* __FM_IOCTLS_H */ diff --git a/sys/contrib/ncsw/inc/ioctls/fm_pcd_ioctls.h b/sys/contrib/ncsw/inc/ioctls/fm_pcd_ioctls.h new file mode 100644 index 000000000000..dfb36d0ace15 --- /dev/null +++ b/sys/contrib/ncsw/inc/ioctls/fm_pcd_ioctls.h @@ -0,0 +1,2982 @@ +/* Copyright (c) 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2021 NXP + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +/****************************************************************************** + @File fm_pcd_ioctls.h + + @Description FM PCD ... +*//***************************************************************************/ +#ifndef __FM_PCD_IOCTLS_H +#define __FM_PCD_IOCTLS_H + +#include "net_ioctls.h" +#include "fm_ioctls.h" + + +/**************************************************************************//** + + @Group lnx_ioctl_FM_grp Frame Manager Linux IOCTL API + + @Description Frame Manager Linux ioctls definitions and enums + + @{ +*//***************************************************************************/ + +/**************************************************************************//** + @Group lnx_ioctl_FM_PCD_grp FM PCD + + @Description Frame Manager PCD API functions, definitions and enums + + The FM PCD module is responsible for the initialization of all + global classifying FM modules. This includes the parser general and + common registers, the key generator global and common registers, + and the policer global and common registers. + In addition, the FM PCD SW module will initialize all required + key generator schemes, coarse classification flows, and policer + profiles. When an FM module is configured to work with one of these + entities, it will register to it using the FM PORT API. The PCD + module will manage the PCD resources - i.e. resource management of + KeyGen schemes, etc. + + @{ +*//***************************************************************************/ + +/**************************************************************************//** + @Collection General PCD defines +*//***************************************************************************/ +#define IOC_FM_PCD_MAX_NUM_OF_PRIVATE_HDRS 2 /**< Number of units/headers saved for user */ + +#define IOC_FM_PCD_PRS_NUM_OF_HDRS 16 /**< Number of headers supported by HW parser */ +#define IOC_FM_PCD_MAX_NUM_OF_DISTINCTION_UNITS (32 - IOC_FM_PCD_MAX_NUM_OF_PRIVATE_HDRS) + /**< Number of distinction units is limited by + register size (32 bits) minus reserved bits + for private headers. */ +#define IOC_FM_PCD_MAX_NUM_OF_INTERCHANGEABLE_HDRS 4 /**< Maximum number of interchangeable headers + in a distinction unit */ +#define IOC_FM_PCD_KG_NUM_OF_GENERIC_REGS 8 /**< Total number of generic KeyGen registers */ +#define IOC_FM_PCD_KG_MAX_NUM_OF_EXTRACTS_PER_KEY 35 /**< Max number allowed on any configuration; + For HW implementation reasons, in most + cases less than this will be allowed; The + driver will return an initialization error + if resource is unavailable. */ +#define IOC_FM_PCD_KG_NUM_OF_EXTRACT_MASKS 4 /**< Total number of masks allowed on KeyGen extractions. */ +#define IOC_FM_PCD_KG_NUM_OF_DEFAULT_GROUPS 16 /**< Number of default value logical groups */ + +#define IOC_FM_PCD_PRS_NUM_OF_LABELS 32 /**< Maximum number of SW parser labels */ +#define IOC_FM_PCD_SW_PRS_SIZE 0x00000800 /**< Total size of SW parser area */ + +#define IOC_FM_PCD_MAX_MANIP_INSRT_TEMPLATE_SIZE 128 /**< Maximum size of insertion template for + insert manipulation */ + +#define IOC_FM_PCD_FRM_REPLIC_MAX_NUM_OF_ENTRIES 64 /**< Maximum possible entries for frame replicator group */ +/* @} */ + +/**************************************************************************//** + @Group lnx_ioctl_FM_PCD_init_grp FM PCD Initialization Unit + + @Description Frame Manager PCD Initialization Unit API + + @{ +*//***************************************************************************/ + +/**************************************************************************//** + @Description PCD counters + (must match enum e_FmPcdCounters defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef enum ioc_fm_pcd_counters { + e_IOC_FM_PCD_KG_COUNTERS_TOTAL, /**< KeyGen counter */ + e_IOC_FM_PCD_PLCR_COUNTERS_RED, /**< Policer counter - counts the total number of RED packets that exit the Policer. */ + e_IOC_FM_PCD_PLCR_COUNTERS_YELLOW, /**< Policer counter - counts the total number of YELLOW packets that exit the Policer. */ + e_IOC_FM_PCD_PLCR_COUNTERS_RECOLORED_TO_RED, /**< Policer counter - counts the number of packets that changed color to RED by the Policer; + This is a subset of e_IOC_FM_PCD_PLCR_COUNTERS_RED packet count, indicating active color changes. */ + e_IOC_FM_PCD_PLCR_COUNTERS_RECOLORED_TO_YELLOW, /**< Policer counter - counts the number of packets that changed color to YELLOW by the Policer; + This is a subset of e_IOC_FM_PCD_PLCR_COUNTERS_YELLOW packet count, indicating active color changes. */ + e_IOC_FM_PCD_PLCR_COUNTERS_TOTAL, /**< Policer counter - counts the total number of packets passed in the Policer. */ + e_IOC_FM_PCD_PLCR_COUNTERS_LENGTH_MISMATCH, /**< Policer counter - counts the number of packets with length mismatch. */ + e_IOC_FM_PCD_PRS_COUNTERS_PARSE_DISPATCH, /**< Parser counter - counts the number of times the parser block is dispatched. */ + e_IOC_FM_PCD_PRS_COUNTERS_L2_PARSE_RESULT_RETURNED, /**< Parser counter - counts the number of times L2 parse result is returned (including errors). */ + e_IOC_FM_PCD_PRS_COUNTERS_L3_PARSE_RESULT_RETURNED, /**< Parser counter - counts the number of times L3 parse result is returned (including errors). */ + e_IOC_FM_PCD_PRS_COUNTERS_L4_PARSE_RESULT_RETURNED, /**< Parser counter - counts the number of times L4 parse result is returned (including errors). */ + e_IOC_FM_PCD_PRS_COUNTERS_SHIM_PARSE_RESULT_RETURNED, /**< Parser counter - counts the number of times SHIM parse result is returned (including errors). */ + e_IOC_FM_PCD_PRS_COUNTERS_L2_PARSE_RESULT_RETURNED_WITH_ERR, /**< Parser counter - counts the number of times L2 parse result is returned with errors. */ + e_IOC_FM_PCD_PRS_COUNTERS_L3_PARSE_RESULT_RETURNED_WITH_ERR, /**< Parser counter - counts the number of times L3 parse result is returned with errors. */ + e_IOC_FM_PCD_PRS_COUNTERS_L4_PARSE_RESULT_RETURNED_WITH_ERR, /**< Parser counter - counts the number of times L4 parse result is returned with errors. */ + e_IOC_FM_PCD_PRS_COUNTERS_SHIM_PARSE_RESULT_RETURNED_WITH_ERR, /**< Parser counter - counts the number of times SHIM parse result is returned with errors. */ + e_IOC_FM_PCD_PRS_COUNTERS_SOFT_PRS_CYCLES, /**< Parser counter - counts the number of cycles spent executing soft parser instruction (including stall cycles). */ + e_IOC_FM_PCD_PRS_COUNTERS_SOFT_PRS_STALL_CYCLES, /**< Parser counter - counts the number of cycles stalled waiting for parser internal memory reads while executing soft parser instruction. */ + e_IOC_FM_PCD_PRS_COUNTERS_HARD_PRS_CYCLE_INCL_STALL_CYCLES, /**< Parser counter - counts the number of cycles spent executing hard parser (including stall cycles). */ + e_IOC_FM_PCD_PRS_COUNTERS_MURAM_READ_CYCLES, /**< MURAM counter - counts the number of cycles while performing FMan Memory read. */ + e_IOC_FM_PCD_PRS_COUNTERS_MURAM_READ_STALL_CYCLES, /**< MURAM counter - counts the number of cycles stalled while performing FMan Memory read. */ + e_IOC_FM_PCD_PRS_COUNTERS_MURAM_WRITE_CYCLES, /**< MURAM counter - counts the number of cycles while performing FMan Memory write. */ + e_IOC_FM_PCD_PRS_COUNTERS_MURAM_WRITE_STALL_CYCLES, /**< MURAM counter - counts the number of cycles stalled while performing FMan Memory write. */ + e_IOC_FM_PCD_PRS_COUNTERS_FPM_COMMAND_STALL_CYCLES /**< FPM counter - counts the number of cycles stalled while performing a FPM Command. */ +} ioc_fm_pcd_counters; + +/**************************************************************************//** + @Description PCD interrupts + (must match enum e_FmPcdExceptions defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef enum ioc_fm_pcd_exceptions { + e_IOC_FM_PCD_KG_EXCEPTION_DOUBLE_ECC, /**< KeyGen double-bit ECC error is detected on internal memory read access. */ + e_IOC_FM_PCD_KG_EXCEPTION_KEYSIZE_OVERFLOW, /**< KeyGen scheme configuration error indicating a key size larger than 56 bytes. */ + e_IOC_FM_PCD_PLCR_EXCEPTION_DOUBLE_ECC, /**< Policer double-bit ECC error has been detected on PRAM read access. */ + e_IOC_FM_PCD_PLCR_EXCEPTION_INIT_ENTRY_ERROR, /**< Policer access to a non-initialized profile has been detected. */ + e_IOC_FM_PCD_PLCR_EXCEPTION_PRAM_SELF_INIT_COMPLETE, /**< Policer RAM self-initialization complete */ + e_IOC_FM_PCD_PLCR_EXCEPTION_ATOMIC_ACTION_COMPLETE, /**< Policer atomic action complete */ + e_IOC_FM_PCD_PRS_EXCEPTION_DOUBLE_ECC, /**< Parser double-bit ECC error */ + e_IOC_FM_PCD_PRS_EXCEPTION_SINGLE_ECC /**< Parser single-bit ECC error */ +} ioc_fm_pcd_exceptions; + +/** @} */ /* end of lnx_ioctl_FM_PCD_init_grp group */ + + +/**************************************************************************//** + @Group lnx_ioctl_FM_PCD_Runtime_grp FM PCD Runtime Unit + + @Description Frame Manager PCD Runtime Unit + + The runtime control allows creation of PCD infrastructure modules + such as Network Environment Characteristics, Classification Plan + Groups and Coarse Classification Trees. + It also allows on-the-fly initialization, modification and removal + of PCD modules such as KeyGen schemes, coarse classification nodes + and Policer profiles. + + In order to explain the programming model of the PCD driver interface + a few terms should be explained, and will be used below. + - Distinction Header - One of the 16 protocols supported by the FM parser, + or one of the SHIM headers (1 or 2). May be a header with a special + option (see below). + - Interchangeable Headers Group - This is a group of Headers recognized + by either one of them. For example, if in a specific context the user + chooses to treat IPv4 and IPV6 in the same way, they may create an + interchangeable Headers Unit consisting of these 2 headers. + - A Distinction Unit - a Distinction Header or an Interchangeable Headers + Group. + - Header with special option - applies to Ethernet, MPLS, VLAN, IPv4 and + IPv6, includes multicast, broadcast and other protocol specific options. + In terms of hardware it relates to the options available in the classification + plan. + - Network Environment Characteristics - a set of Distinction Units that define + the total recognizable header selection for a certain environment. This is + NOT the list of all headers that will ever appear in a flow, but rather + everything that needs distinction in a flow, where distinction is made by KeyGen + schemes and coarse classification action descriptors. + + The PCD runtime modules initialization is done in stages. The first stage after + initializing the PCD module itself is to establish a Network Flows Environment + Definition. The application may choose to establish one or more such environments. + Later, when needed, the application will have to state, for some of its modules, + to which single environment it belongs. + + @{ +*//***************************************************************************/ + + +/**************************************************************************//** + @Description structure for FM counters +*//***************************************************************************/ +typedef struct ioc_fm_pcd_counters_params_t { + ioc_fm_pcd_counters cnt; /**< The requested counter */ + uint32_t val; /**< The requested value to get/set from/into the counter */ +} ioc_fm_pcd_counters_params_t; + +/**************************************************************************//** + @Description structure for FM exception definitios +*//***************************************************************************/ +typedef struct ioc_fm_pcd_exception_params_t { + ioc_fm_pcd_exceptions exception; /**< The requested exception */ + bool enable; /**< TRUE to enable interrupt, FALSE to mask it. */ +} ioc_fm_pcd_exception_params_t; + +/**************************************************************************//** + @Description A structure for SW parser labels + (must be identical to struct t_FmPcdPrsLabelParams defined in fm_pcd_ext.h) + *//***************************************************************************/ +typedef struct ioc_fm_pcd_prs_label_params_t { + uint32_t instruction_offset; /**< SW parser label instruction offset (2 bytes + resolution), relative to Parser RAM. */ + ioc_net_header_type hdr; /**< The existence of this header will invoke + the SW parser code. */ + uint8_t index_per_hdr; /**< Normally 0, if more than one SW parser + attachments for the same header, use this + index to distinguish between them. */ +} ioc_fm_pcd_prs_label_params_t; + +/**************************************************************************//** + @Description A structure for SW parser + (Must match struct t_FmPcdPrsSwParams defined in fm_pcd_ext.h) + *//***************************************************************************/ +typedef struct ioc_fm_pcd_prs_sw_params_t { + bool override; /**< FALSE to invoke a check that nothing else + was loaded to this address, including + internal patches. + TRUE to override any existing code.*/ + uint32_t size; /**< SW parser code size */ + uint16_t base; /**< SW parser base (in instruction counts! + must be larger than 0x20)*/ + uint8_t *p_code; /**< SW parser code */ + uint32_t sw_prs_data_params[IOC_FM_PCD_PRS_NUM_OF_HDRS]; + /**< SW parser data (parameters) */ + uint8_t num_of_labels; /**< Number of labels for SW parser. */ + ioc_fm_pcd_prs_label_params_t labels_table[IOC_FM_PCD_PRS_NUM_OF_LABELS]; + /**< SW parser labels table, + containing num_of_labels entries */ +} ioc_fm_pcd_prs_sw_params_t; + +/**************************************************************************//** + @Description A structure to set the a KeyGen default value + *//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_dflt_value_params_t { + uint8_t valueId; /**< 0,1 - one of 2 global default values */ + uint32_t value; /**< The requested default value */ +} ioc_fm_pcd_kg_dflt_value_params_t; + + +/**************************************************************************//** + @Function FM_PCD_Enable + + @Description This routine should be called after PCD is initialized for enabling all + PCD engines according to their existing configuration. + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only when PCD is disabled. +*//***************************************************************************/ +#define FM_PCD_IOC_ENABLE _IO(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(1)) + +/**************************************************************************//** + @Function FM_PCD_Disable + + @Description This routine may be called when PCD is enabled in order to + disable all PCD engines. It may be called + only when none of the ports in the system are using the PCD. + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only when PCD is enabled. +*//***************************************************************************/ +#define FM_PCD_IOC_DISABLE _IO(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(2)) + + /**************************************************************************//** + @Function FM_PCD_PrsLoadSw + + @Description This routine may be called only when all ports in the + system are actively using the classification plan scheme. + In such cases it is recommended in order to save resources. + The driver automatically saves 8 classification plans for + ports that do NOT use the classification plan mechanism, to + avoid this (in order to save those entries) this routine may + be called. + + @Param[in] ioc_fm_pcd_prs_sw_params_t A pointer to the image of the software parser code. + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only when PCD is disabled. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_PRS_LOAD_SW_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(3), ioc_compat_fm_pcd_prs_sw_params_t) +#endif +#define FM_PCD_IOC_PRS_LOAD_SW _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(3), ioc_fm_pcd_prs_sw_params_t) + +/**************************************************************************//** + @Function FM_PCD_KgSetDfltValue + + @Description Calling this routine sets a global default value to be used + by the KeyGen when parser does not recognize a required + field/header. + By default default values are 0. + + @Param[in] ioc_fm_pcd_kg_dflt_value_params_t A pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only when PCD is disabled. +*//***************************************************************************/ +#define FM_PCD_IOC_KG_SET_DFLT_VALUE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(6), ioc_fm_pcd_kg_dflt_value_params_t) + +/**************************************************************************//** + @Function FM_PCD_KgSetAdditionalDataAfterParsing + + @Description Calling this routine allows the keygen to access data past + the parser finishing point. + + @Param[in] uint8_t payload-offset; the number of bytes beyond the parser location. + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only when PCD is disabled. +*//***************************************************************************/ +#define FM_PCD_IOC_KG_SET_ADDITIONAL_DATA_AFTER_PARSING _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(7), uint8_t) + +/**************************************************************************//** + @Function FM_PCD_SetException + + @Description Calling this routine enables/disables PCD interrupts. + + @Param[in] ioc_fm_pcd_exception_params_t Arguments struct with exception to be enabled/disabled. + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#define FM_PCD_IOC_SET_EXCEPTION _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(8), ioc_fm_pcd_exception_params_t) + +/**************************************************************************//** + @Function FM_PCD_GetCounter + + @Description Reads one of the FM PCD counters. + + @Param[in,out] ioc_fm_pcd_counters_params_t The requested counter parameters. + + @Return 0 on success; Error code otherwise. + + @Cautions Note that it is user's responsibilty to call this routine only + for enabled counters, and there will be no indication if a + disabled counter is accessed. +*//***************************************************************************/ +#define FM_PCD_IOC_GET_COUNTER _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(9), ioc_fm_pcd_counters_params_t) + +/**************************************************************************//** + + @Function FM_PCD_KgSchemeGetCounter + + @Description Reads scheme packet counter. + + @Param[in] h_Scheme scheme handle as returned by FM_PCD_KgSchemeSet(). + + @Return Counter's current value. + + @Cautions Allowed only following FM_PCD_Init() & FM_PCD_KgSchemeSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_KG_SCHEME_GET_CNTR_COMPAT _IOR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(4), ioc_compat_fm_pcd_kg_scheme_spc_t) +#endif +#define FM_PCD_IOC_KG_SCHEME_GET_CNTR _IOR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(4), ioc_fm_pcd_kg_scheme_spc_t) + +#if 0 +TODO: unused IOCTL +/**************************************************************************//** + @Function FM_PCD_ModifyCounter + + @Description Writes a value to an enabled counter. Use "0" to reset the counter. + + @Param[in] ioc_fm_pcd_counters_params_t - The requested counter parameters. + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#define FM_PCD_IOC_MODIFY_COUNTER _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(10), ioc_fm_pcd_counters_params_t) +#define FM_PCD_IOC_SET_COUNTER FM_PCD_IOC_MODIFY_COUNTER +#endif + +/**************************************************************************//** + @Function FM_PCD_ForceIntr + + @Description Causes an interrupt event on the requested source. + + @Param[in] ioc_fm_pcd_exceptions - An exception to be forced. + + @Return 0 on success; error code if the exception is not enabled, + or is not able to create interrupt. +*//***************************************************************************/ +#define FM_PCD_IOC_FORCE_INTR _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(11), ioc_fm_pcd_exceptions) + +/**************************************************************************//** + @Collection Definitions of coarse classification parameters as required by KeyGen + (when coarse classification is the next engine after this scheme). +*//***************************************************************************/ +#define IOC_FM_PCD_MAX_NUM_OF_CC_TREES 8 +#define IOC_FM_PCD_MAX_NUM_OF_CC_GROUPS 16 +#define IOC_FM_PCD_MAX_NUM_OF_CC_UNITS 4 +#define IOC_FM_PCD_MAX_NUM_OF_KEYS 256 +#define IOC_FM_PCD_MAX_NUM_OF_FLOWS (4*KILOBYTE) +#define IOC_FM_PCD_MAX_SIZE_OF_KEY 56 +#define IOC_FM_PCD_MAX_NUM_OF_CC_ENTRIES_IN_GRP 16 +#define IOC_FM_PCD_LAST_KEY_INDEX 0xffff +#define IOC_FM_PCD_MANIP_DSCP_VALUES 64 +/* @} */ + +/**************************************************************************//** + @Collection A set of definitions to allow protocol + special option description. +*//***************************************************************************/ +typedef uint32_t ioc_protocol_opt_t; /**< A general type to define a protocol option. */ + +typedef ioc_protocol_opt_t ioc_eth_protocol_opt_t; /**< Ethernet protocol options. */ +#define IOC_ETH_BROADCAST 0x80000000 /**< Ethernet Broadcast. */ +#define IOC_ETH_MULTICAST 0x40000000 /**< Ethernet Multicast. */ + +typedef ioc_protocol_opt_t ioc_vlan_protocol_opt_t; /**< Vlan protocol options. */ +#define IOC_VLAN_STACKED 0x20000000 /**< Stacked VLAN. */ + +typedef ioc_protocol_opt_t ioc_mpls_protocol_opt_t; /**< MPLS protocol options. */ +#define IOC_MPLS_STACKED 0x10000000 /**< Stacked MPLS. */ + +typedef ioc_protocol_opt_t ioc_ipv4_protocol_opt_t; /**< IPv4 protocol options. */ +#define IOC_IPV4_BROADCAST_1 0x08000000 /**< IPv4 Broadcast. */ +#define IOC_IPV4_MULTICAST_1 0x04000000 /**< IPv4 Multicast. */ +#define IOC_IPV4_UNICAST_2 0x02000000 /**< Tunneled IPv4 - Unicast. */ +#define IOC_IPV4_MULTICAST_BROADCAST_2 0x01000000 /**< Tunneled IPv4 - Broadcast/Multicast. */ + +#define IOC_IPV4_FRAG_1 0x00000008 /**< IPV4 reassembly option. + IPV4 Reassembly manipulation requires network + environment with IPV4 header and IPV4_FRAG_1 option */ + +typedef ioc_protocol_opt_t ioc_ipv6_protocol_opt_t; /**< IPv6 protocol options. */ +#define IOC_IPV6_MULTICAST_1 0x00800000 /**< IPv6 Multicast. */ +#define IOC_IPV6_UNICAST_2 0x00400000 /**< Tunneled IPv6 - Unicast. */ +#define IOC_IPV6_MULTICAST_2 0x00200000 /**< Tunneled IPv6 - Multicast. */ + +#define IOC_IPV6_FRAG_1 0x00000004 /**< IPV6 reassembly option. + IPV6 Reassembly manipulation requires network + environment with IPV6 header and IPV6_FRAG_1 option */ +typedef ioc_protocol_opt_t ioc_capwap_protocol_opt_t; /**< CAPWAP protocol options. */ +#define CAPWAP_FRAG_1 0x00000008 /**< CAPWAP reassembly option. + CAPWAP Reassembly manipulation requires network + environment with CAPWAP header and CAPWAP_FRAG_1 option; + in case where fragment found, the fragment-extension offset + may be found at 'shim2' (in parser-result). */ +/* @} */ + +#define IOC_FM_PCD_MANIP_MAX_HDR_SIZE 256 +#define IOC_FM_PCD_MANIP_DSCP_TO_VLAN_TRANS 64 +/**************************************************************************//** + @Collection A set of definitions to support Header Manipulation selection. +*//***************************************************************************/ +typedef uint32_t ioc_hdr_manip_flags_t; /**< A general type to define a HMan update command flags. */ + +typedef ioc_hdr_manip_flags_t ioc_ipv4_hdr_manip_update_flags_t; /**< IPv4 protocol HMan update command flags. */ + +#define IOC_HDR_MANIP_IPV4_TOS 0x80000000 /**< update TOS with the given value ('tos' field + of ioc_fm_pcd_manip_hdr_field_update_ipv4_t) */ +#define IOC_HDR_MANIP_IPV4_ID 0x40000000 /**< update IP ID with the given value ('id' field + of ioc_fm_pcd_manip_hdr_field_update_ipv4_t) */ +#define IOC_HDR_MANIP_IPV4_TTL 0x20000000 /**< Decrement TTL by 1 */ +#define IOC_HDR_MANIP_IPV4_SRC 0x10000000 /**< update IP source address with the given value + ('src' field of ioc_fm_pcd_manip_hdr_field_update_ipv4_t) */ +#define IOC_HDR_MANIP_IPV4_DST 0x08000000 /**< update IP destination address with the given value + ('dst' field of ioc_fm_pcd_manip_hdr_field_update_ipv4_t) */ + +typedef ioc_hdr_manip_flags_t ioc_ipv6_hdr_manip_update_flags_t; /**< IPv6 protocol HMan update command flags. */ + +#define IOC_HDR_MANIP_IPV6_TC 0x80000000 /**< update Traffic Class address with the given value + ('traffic_class' field of ioc_fm_pcd_manip_hdr_field_update_ipv6_t) */ +#define IOC_HDR_MANIP_IPV6_HL 0x40000000 /**< Decrement Hop Limit by 1 */ +#define IOC_HDR_MANIP_IPV6_SRC 0x20000000 /**< update IP source address with the given value + ('src' field of ioc_fm_pcd_manip_hdr_field_update_ipv6_t) */ +#define IOC_HDR_MANIP_IPV6_DST 0x10000000 /**< update IP destination address with the given value + ('dst' field of ioc_fm_pcd_manip_hdr_field_update_ipv6_t) */ + +typedef ioc_hdr_manip_flags_t ioc_tcp_udp_hdr_manip_update_flags_t;/**< TCP/UDP protocol HMan update command flags. */ + +#define IOC_HDR_MANIP_TCP_UDP_SRC 0x80000000 /**< update TCP/UDP source address with the given value + ('src' field of ioc_fm_pcd_manip_hdr_field_update_tcp_udp_t) */ +#define IOC_HDR_MANIP_TCP_UDP_DST 0x40000000 /**< update TCP/UDP destination address with the given value + ('dst' field of ioc_fm_pcd_manip_hdr_field_update_tcp_udp_t) */ +#define IOC_HDR_MANIP_TCP_UDP_CHECKSUM 0x20000000 /**< update TCP/UDP checksum */ + +/* @} */ + +/**************************************************************************//** + @Description A type used for returning the order of the key extraction. + each value in this array represents the index of the extraction + command as defined by the user in the initialization extraction array. + The valid size of this array is the user define number of extractions + required (also marked by the second '0' in this array). +*//***************************************************************************/ +typedef uint8_t ioc_fm_pcd_kg_key_order_t [IOC_FM_PCD_KG_MAX_NUM_OF_EXTRACTS_PER_KEY]; + +/**************************************************************************//** + @Description All PCD engines + (must match enum e_FmPcdEngine defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef enum ioc_fm_pcd_engine { + e_IOC_FM_PCD_INVALID = 0, /**< Invalid PCD engine */ + e_IOC_FM_PCD_DONE, /**< No PCD Engine indicated */ + e_IOC_FM_PCD_KG, /**< KeyGen */ + e_IOC_FM_PCD_CC, /**< Coarse Classifier */ + e_IOC_FM_PCD_PLCR, /**< Policer */ + e_IOC_FM_PCD_PRS, /**< Parser */ + e_IOC_FM_PCD_FR, /**< Frame Replicator */ + e_IOC_FM_PCD_HASH /**< Hash Table */ +} ioc_fm_pcd_engine; + +/**************************************************************************//** + @Description An enum for selecting extraction by header types + (Must match enum e_FmPcdExtractByHdrType defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef enum ioc_fm_pcd_extract_by_hdr_type { + e_IOC_FM_PCD_EXTRACT_FROM_HDR, /**< Extract bytes from header */ + e_IOC_FM_PCD_EXTRACT_FROM_FIELD, /**< Extract bytes from header field */ + e_IOC_FM_PCD_EXTRACT_FULL_FIELD /**< Extract a full field */ +} ioc_fm_pcd_extract_by_hdr_type; + +/**************************************************************************//** + @Description An enum for selecting extraction source (when it is not the header) + (Must match enum e_FmPcdExtractFrom defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef enum ioc_fm_pcd_extract_from { + e_IOC_FM_PCD_EXTRACT_FROM_FRAME_START, /**< KG & CC: Extract from beginning of frame */ + e_IOC_FM_PCD_EXTRACT_FROM_DFLT_VALUE, /**< KG only: Extract from a default value */ + e_IOC_FM_PCD_EXTRACT_FROM_CURR_END_OF_PARSE, /**< KG only: Extract from the point where parsing had finished */ + e_IOC_FM_PCD_EXTRACT_FROM_KEY, /**< CC only: Field where saved KEY */ + e_IOC_FM_PCD_EXTRACT_FROM_HASH, /**< CC only: Field where saved HASH */ + e_IOC_FM_PCD_EXTRACT_FROM_PARSE_RESULT, /**< KG & CC: Extract from the parser result */ + e_IOC_FM_PCD_EXTRACT_FROM_ENQ_FQID, /**< KG & CC: Extract from enqueue FQID */ + e_IOC_FM_PCD_EXTRACT_FROM_FLOW_ID /**< CC only: Field where saved Dequeue FQID */ +} ioc_fm_pcd_extract_from; + +/**************************************************************************//** + @Description An enum for selecting extraction type +*//***************************************************************************/ +typedef enum ioc_fm_pcd_extract_type { + e_IOC_FM_PCD_EXTRACT_BY_HDR, /**< Extract according to header */ + e_IOC_FM_PCD_EXTRACT_NON_HDR, /**< Extract from data that is not the header */ + e_IOC_FM_PCD_KG_EXTRACT_PORT_PRIVATE_INFO /**< Extract private info as specified by user */ +} ioc_fm_pcd_extract_type; + +/**************************************************************************//** + @Description An enum for selecting a default +*//***************************************************************************/ +typedef enum ioc_fm_pcd_kg_extract_dflt_select { + e_IOC_FM_PCD_KG_DFLT_GBL_0, /**< Default selection is KG register 0 */ + e_IOC_FM_PCD_KG_DFLT_GBL_1, /**< Default selection is KG register 1 */ + e_IOC_FM_PCD_KG_DFLT_PRIVATE_0, /**< Default selection is a per scheme register 0 */ + e_IOC_FM_PCD_KG_DFLT_PRIVATE_1, /**< Default selection is a per scheme register 1 */ + e_IOC_FM_PCD_KG_DFLT_ILLEGAL /**< Illegal selection */ +} ioc_fm_pcd_kg_extract_dflt_select; + +/**************************************************************************//** + @Description Enumeration type defining all default groups - each group shares + a default value, one of four user-initialized values. +*//***************************************************************************/ +typedef enum ioc_fm_pcd_kg_known_fields_dflt_types { + e_IOC_FM_PCD_KG_MAC_ADDR, /**< MAC Address */ + e_IOC_FM_PCD_KG_TCI, /**< TCI field */ + e_IOC_FM_PCD_KG_ENET_TYPE, /**< ENET Type */ + e_IOC_FM_PCD_KG_PPP_SESSION_ID, /**< PPP Session id */ + e_IOC_FM_PCD_KG_PPP_PROTOCOL_ID, /**< PPP Protocol id */ + e_IOC_FM_PCD_KG_MPLS_LABEL, /**< MPLS label */ + e_IOC_FM_PCD_KG_IP_ADDR, /**< IP addr */ + e_IOC_FM_PCD_KG_PROTOCOL_TYPE, /**< Protocol type */ + e_IOC_FM_PCD_KG_IP_TOS_TC, /**< TOS or TC */ + e_IOC_FM_PCD_KG_IPV6_FLOW_LABEL, /**< IPV6 flow label */ + e_IOC_FM_PCD_KG_IPSEC_SPI, /**< IPSEC SPI */ + e_IOC_FM_PCD_KG_L4_PORT, /**< L4 Port */ + e_IOC_FM_PCD_KG_TCP_FLAG, /**< TCP Flag */ + e_IOC_FM_PCD_KG_GENERIC_FROM_DATA, /**< grouping implemented by SW, + any data extraction that is not the full + field described above */ + e_IOC_FM_PCD_KG_GENERIC_FROM_DATA_NO_V, /**< grouping implemented by SW, + any data extraction without validation */ + e_IOC_FM_PCD_KG_GENERIC_NOT_FROM_DATA /**< grouping implemented by SW, + extraction from parser result or + direct use of default value */ +} ioc_fm_pcd_kg_known_fields_dflt_types; + +/**************************************************************************//** + @Description Enumeration type for defining header index for scenarios with + multiple (tunneled) headers +*//***************************************************************************/ +typedef enum ioc_fm_pcd_hdr_index { + e_IOC_FM_PCD_HDR_INDEX_NONE = 0, /**< used when multiple headers not used, also + to specify regular IP (not tunneled). */ + e_IOC_FM_PCD_HDR_INDEX_1, /**< may be used for VLAN, MPLS, tunneled IP */ + e_IOC_FM_PCD_HDR_INDEX_2, /**< may be used for MPLS, tunneled IP */ + e_IOC_FM_PCD_HDR_INDEX_3, /**< may be used for MPLS */ + e_IOC_FM_PCD_HDR_INDEX_LAST = 0xFF /**< may be used for VLAN, MPLS */ +} ioc_fm_pcd_hdr_index; + +/**************************************************************************//** + @Description Enumeration type for selecting the policer profile functional type +*//***************************************************************************/ +typedef enum ioc_fm_pcd_profile_type_selection { + e_IOC_FM_PCD_PLCR_PORT_PRIVATE, /**< Port dedicated profile */ + e_IOC_FM_PCD_PLCR_SHARED /**< Shared profile (shared within partition) */ +} ioc_fm_pcd_profile_type_selection; + +/**************************************************************************//** + @Description Enumeration type for selecting the policer profile algorithm +*//***************************************************************************/ +typedef enum ioc_fm_pcd_plcr_algorithm_selection { + e_IOC_FM_PCD_PLCR_PASS_THROUGH, /**< Policer pass through */ + e_IOC_FM_PCD_PLCR_RFC_2698, /**< Policer algorithm RFC 2698 */ + e_IOC_FM_PCD_PLCR_RFC_4115 /**< Policer algorithm RFC 4115 */ +} ioc_fm_pcd_plcr_algorithm_selection; + +/**************************************************************************//** + @Description Enumeration type for selecting a policer profile color mode +*//***************************************************************************/ +typedef enum ioc_fm_pcd_plcr_color_mode { + e_IOC_FM_PCD_PLCR_COLOR_BLIND, /**< Color blind */ + e_IOC_FM_PCD_PLCR_COLOR_AWARE /**< Color aware */ +} ioc_fm_pcd_plcr_color_mode; + +/**************************************************************************//** + @Description Enumeration type for selecting a policer profile color +*//***************************************************************************/ +typedef enum ioc_fm_pcd_plcr_color { + e_IOC_FM_PCD_PLCR_GREEN, /**< Green */ + e_IOC_FM_PCD_PLCR_YELLOW, /**< Yellow */ + e_IOC_FM_PCD_PLCR_RED, /**< Red */ + e_IOC_FM_PCD_PLCR_OVERRIDE /**< Color override */ +} ioc_fm_pcd_plcr_color; + +/**************************************************************************//** + @Description Enumeration type for selecting the policer profile packet frame length selector +*//***************************************************************************/ +typedef enum ioc_fm_pcd_plcr_frame_length_select { + e_IOC_FM_PCD_PLCR_L2_FRM_LEN, /**< L2 frame length */ + e_IOC_FM_PCD_PLCR_L3_FRM_LEN, /**< L3 frame length */ + e_IOC_FM_PCD_PLCR_L4_FRM_LEN, /**< L4 frame length */ + e_IOC_FM_PCD_PLCR_FULL_FRM_LEN /**< Full frame length */ +} ioc_fm_pcd_plcr_frame_length_select; + +/**************************************************************************//** + @Description Enumeration type for selecting roll-back frame +*//***************************************************************************/ +typedef enum ioc_fm_pcd_plcr_roll_back_frame_select { + e_IOC_FM_PCD_PLCR_ROLLBACK_L2_FRM_LEN, /**< Rollback L2 frame length */ + e_IOC_FM_PCD_PLCR_ROLLBACK_FULL_FRM_LEN /**< Rollback Full frame length */ +} ioc_fm_pcd_plcr_roll_back_frame_select; + +/**************************************************************************//** + @Description Enumeration type for selecting the policer profile packet or byte mode +*//***************************************************************************/ +typedef enum ioc_fm_pcd_plcr_rate_mode { + e_IOC_FM_PCD_PLCR_BYTE_MODE, /**< Byte mode */ + e_IOC_FM_PCD_PLCR_PACKET_MODE /**< Packet mode */ +} ioc_fm_pcd_plcr_rate_mode; + +/**************************************************************************//** + @Description Enumeration type for defining action of frame +*//***************************************************************************/ +typedef enum ioc_fm_pcd_done_action { + e_IOC_FM_PCD_ENQ_FRAME = 0, /**< Enqueue frame */ + e_IOC_FM_PCD_DROP_FRAME /**< Drop frame */ +} ioc_fm_pcd_done_action; + +/**************************************************************************//** + @Description Enumeration type for selecting the policer counter +*//***************************************************************************/ +typedef enum ioc_fm_pcd_plcr_profile_counters { + e_IOC_FM_PCD_PLCR_PROFILE_GREEN_PACKET_TOTAL_COUNTER, /**< Green packets counter */ + e_IOC_FM_PCD_PLCR_PROFILE_YELLOW_PACKET_TOTAL_COUNTER, /**< Yellow packets counter */ + e_IOC_FM_PCD_PLCR_PROFILE_RED_PACKET_TOTAL_COUNTER, /**< Red packets counter */ + e_IOC_FM_PCD_PLCR_PROFILE_RECOLOURED_YELLOW_PACKET_TOTAL_COUNTER, /**< Recolored yellow packets counter */ + e_IOC_FM_PCD_PLCR_PROFILE_RECOLOURED_RED_PACKET_TOTAL_COUNTER /**< Recolored red packets counter */ +} ioc_fm_pcd_plcr_profile_counters; + +/**************************************************************************//** + @Description Enumeration type for selecting the PCD action after extraction +*//***************************************************************************/ +typedef enum ioc_fm_pcd_action { + e_IOC_FM_PCD_ACTION_NONE, /**< NONE */ + e_IOC_FM_PCD_ACTION_EXACT_MATCH, /**< Exact match on the selected extraction*/ + e_IOC_FM_PCD_ACTION_INDEXED_LOOKUP /**< Indexed lookup on the selected extraction*/ +} ioc_fm_pcd_action; + +/**************************************************************************//** + @Description Enumeration type for selecting type of insert manipulation +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_insrt_type { + e_IOC_FM_PCD_MANIP_INSRT_GENERIC, /**< Insert according to offset & size */ + e_IOC_FM_PCD_MANIP_INSRT_BY_HDR, /**< Insert according to protocol */ +} ioc_fm_pcd_manip_hdr_insrt_type; + +/**************************************************************************//** + @Description Enumeration type for selecting type of remove manipulation +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_rmv_type { + e_IOC_FM_PCD_MANIP_RMV_GENERIC, /**< Remove according to offset & size */ + e_IOC_FM_PCD_MANIP_RMV_BY_HDR /**< Remove according to offset & size */ +} ioc_fm_pcd_manip_hdr_rmv_type; + +/**************************************************************************//** + @Description An enum for selecting specific L2 fields removal +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_rmv_specific_l2 { + e_IOC_FM_PCD_MANIP_HDR_RMV_ETHERNET, /**< Ethernet/802.3 MAC */ + e_IOC_FM_PCD_MANIP_HDR_RMV_STACKED_QTAGS, /**< stacked QTags */ + e_IOC_FM_PCD_MANIP_HDR_RMV_ETHERNET_AND_MPLS, /**< MPLS and Ethernet/802.3 MAC header until + the header which follows the MPLS header */ + e_IOC_FM_PCD_MANIP_HDR_RMV_MPLS /**< Remove MPLS header (Unlimited MPLS labels) */ +} ioc_fm_pcd_manip_hdr_rmv_specific_l2; + +/**************************************************************************//** + @Description Enumeration type for selecting specific fields updates +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_field_update_type { + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_VLAN, /**< VLAN updates */ + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_IPV4, /**< IPV4 updates */ + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_IPV6, /**< IPV6 updates */ + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_TCP_UDP, /**< TCP_UDP updates */ +} ioc_fm_pcd_manip_hdr_field_update_type; + +/**************************************************************************//** + @Description Enumeration type for selecting VLAN updates +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_field_update_vlan { + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_VLAN_VPRI, /**< Replace VPri of outer most VLAN tag. */ + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_DSCP_TO_VLAN /**< DSCP to VLAN priority bits translation */ +} ioc_fm_pcd_manip_hdr_field_update_vlan; + +/**************************************************************************//** + @Description Enumeration type for selecting specific L2 fields removal +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_insrt_specific_l2 { + e_IOC_FM_PCD_MANIP_HDR_INSRT_MPLS /**< Insert MPLS header (Unlimited MPLS labels) */ +} ioc_fm_pcd_manip_hdr_insrt_specific_l2; + +/**************************************************************************//** + @Description Enumeration type for selecting QoS mapping mode + + Note: In all cases except 'e_FM_PCD_MANIP_HDR_QOS_MAPPING_NONE' + User should instruct the port to read the parser-result +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_qos_mapping_mode { + e_IOC_FM_PCD_MANIP_HDR_QOS_MAPPING_NONE = 0, /**< No mapping, QoS field will not be changed */ + e_IOC_FM_PCD_MANIP_HDR_QOS_MAPPING_AS_IS, /**< QoS field will be overwritten by the last byte in the parser-result. */ +} ioc_fm_pcd_manip_hdr_qos_mapping_mode; + +/**************************************************************************//** + @Description Enumeration type for selecting QoS source + + Note: In all cases except 'e_FM_PCD_MANIP_HDR_QOS_SRC_NONE' + User should left room for the parser-result on input/output buffer + and instruct the port to read/write the parser-result to the buffer (RPD should be set) +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_qos_src { + e_IOC_FM_PCD_MANIP_HDR_QOS_SRC_NONE = 0, /**< TODO */ + e_IOC_FM_PCD_MANIP_HDR_QOS_SRC_USER_DEFINED, /**< QoS will be taken from the last byte in the parser-result. */ +} ioc_fm_pcd_manip_hdr_qos_src; + +/**************************************************************************//** + @Description Enumeration type for selecting type of header insertion +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_insrt_by_hdr_type { + e_IOC_FM_PCD_MANIP_INSRT_BY_HDR_SPECIFIC_L2, /**< Specific L2 fields insertion */ + e_IOC_FM_PCD_MANIP_INSRT_BY_HDR_IP, /**< IP insertion */ + e_IOC_FM_PCD_MANIP_INSRT_BY_HDR_UDP, /**< UDP insertion */ + e_IOC_FM_PCD_MANIP_INSRT_BY_HDR_UDP_LITE, /**< UDP lite insertion */ + e_IOC_FM_PCD_MANIP_INSRT_BY_HDR_CAPWAP /**< CAPWAP insertion */ +} ioc_fm_pcd_manip_hdr_insrt_by_hdr_type; + +/**************************************************************************//** + @Description Enumeration type for selecting specific custom command +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_custom_type { + e_IOC_FM_PCD_MANIP_HDR_CUSTOM_IP_REPLACE, /**< Replace IPv4/IPv6 */ +} ioc_fm_pcd_manip_hdr_custom_type; + +/**************************************************************************//** + @Description Enumeration type for selecting specific custom command +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_custom_ip_replace { + e_IOC_FM_PCD_MANIP_HDR_CUSTOM_REPLACE_IPV4_BY_IPV6, /**< Replace IPv4 by IPv6 */ + e_IOC_FM_PCD_MANIP_HDR_CUSTOM_REPLACE_IPV6_BY_IPV4 /**< Replace IPv6 by IPv4 */ +} ioc_fm_pcd_manip_hdr_custom_ip_replace; + +/**************************************************************************//** + @Description Enumeration type for selecting type of header removal +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_hdr_rmv_by_hdr_type { + e_IOC_FM_PCD_MANIP_RMV_BY_HDR_SPECIFIC_L2 = 0, /**< Specific L2 fields removal */ + e_IOC_FM_PCD_MANIP_RMV_BY_HDR_CAPWAP, /**< CAPWAP removal */ + e_IOC_FM_PCD_MANIP_RMV_BY_HDR_FROM_START, /**< Locate from data that is not the header */ +} ioc_fm_pcd_manip_hdr_rmv_by_hdr_type; + +/**************************************************************************//** + @Description Enumeration type for selecting type of timeout mode +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_reassem_time_out_mode { + e_IOC_FM_PCD_MANIP_TIME_OUT_BETWEEN_FRAMES, /**< Limits the time of the reassembly process + from the first fragment to the last */ + e_IOC_FM_PCD_MANIP_TIME_OUT_BETWEEN_FRAG /**< Limits the time of receiving the fragment */ +} ioc_fm_pcd_manip_reassem_time_out_mode; + +/**************************************************************************//** + @Description Enumeration type for selecting type of WaysNumber mode +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_reassem_ways_number { + e_IOC_FM_PCD_MANIP_ONE_WAY_HASH = 1, /**< One way hash */ + e_IOC_FM_PCD_MANIP_TWO_WAYS_HASH, /**< Two ways hash */ + e_IOC_FM_PCD_MANIP_THREE_WAYS_HASH, /**< Three ways hash */ + e_IOC_FM_PCD_MANIP_FOUR_WAYS_HASH, /**< Four ways hash */ + e_IOC_FM_PCD_MANIP_FIVE_WAYS_HASH, /**< Five ways hash */ + e_IOC_FM_PCD_MANIP_SIX_WAYS_HASH, /**< Six ways hash */ + e_IOC_FM_PCD_MANIP_SEVEN_WAYS_HASH, /**< Seven ways hash */ + e_IOC_FM_PCD_MANIP_EIGHT_WAYS_HASH /**< Eight ways hash */ +} ioc_fm_pcd_manip_reassem_ways_number; + +/**************************************************************************//** + @Description Enumeration type for selecting manipulation type +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_type { + e_IOC_FM_PCD_MANIP_HDR = 0, /**< Header manipulation */ + e_IOC_FM_PCD_MANIP_REASSEM, /**< Reassembly */ + e_IOC_FM_PCD_MANIP_FRAG, /**< Fragmentation */ + e_IOC_FM_PCD_MANIP_SPECIAL_OFFLOAD /**< Special Offloading */ +} ioc_fm_pcd_manip_type; + +/**************************************************************************//** + @Description Enumeration type for selecting type of statistics mode +*//***************************************************************************/ +typedef enum ioc_fm_pcd_cc_stats_mode { + e_IOC_FM_PCD_CC_STATS_MODE_NONE = 0, /**< No statistics support */ + e_IOC_FM_PCD_CC_STATS_MODE_FRAME, /**< Frame count statistics */ + e_IOC_FM_PCD_CC_STATS_MODE_BYTE_AND_FRAME, /**< Byte and frame count statistics */ + e_IOC_FM_PCD_CC_STATS_MODE_RMON, /**< Byte and frame length range count statistics */ +} ioc_fm_pcd_cc_stats_mode; + +/**************************************************************************//** + @Description Enumeration type for determining the action in case an IP packet + is larger than MTU but its DF (Don't Fragment) bit is set. +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_dont_frag_action { + e_IOC_FM_PCD_MANIP_DISCARD_PACKET = 0, /**< Discard packet */ + e_IOC_FM_PCD_MANIP_ENQ_TO_ERR_Q_OR_DISCARD_PACKET = e_IOC_FM_PCD_MANIP_DISCARD_PACKET, + /**< Obsolete, cannot enqueue to error queue; + In practice, selects to discard packets; + Will be removed in the future */ + e_IOC_FM_PCD_MANIP_FRAGMENT_PACKECT, /**< Fragment packet and continue normal processing */ + e_IOC_FM_PCD_MANIP_CONTINUE_WITHOUT_FRAG /**< Continue normal processing without fragmenting the packet */ +} ioc_fm_pcd_manip_dont_frag_action; + +/**************************************************************************//** + @Description Enumeration type for selecting type of special offload manipulation +*//***************************************************************************/ +typedef enum ioc_fm_pcd_manip_special_offload_type { + e_IOC_FM_PCD_MANIP_SPECIAL_OFFLOAD_IPSEC, /**< IPSec offload manipulation */ + e_IOC_FM_PCD_MANIP_SPECIAL_OFFLOAD_CAPWAP /**< CAPWAP offload manipulation */ +} ioc_fm_pcd_manip_special_offload_type; + +/**************************************************************************//** + @Description A union of protocol dependent special options + (Must match union u_FmPcdHdrProtocolOpt defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef union ioc_fm_pcd_hdr_protocol_opt_u { + ioc_eth_protocol_opt_t eth_opt; /**< Ethernet options */ + ioc_vlan_protocol_opt_t vlan_opt; /**< Vlan options */ + ioc_mpls_protocol_opt_t mpls_opt; /**< MPLS options */ + ioc_ipv4_protocol_opt_t ipv4_opt; /**< IPv4 options */ + ioc_ipv6_protocol_opt_t ipv6_opt; /**< IPv6 options */ + ioc_capwap_protocol_opt_t capwap_opt; /**< CAPWAP options */ +} ioc_fm_pcd_hdr_protocol_opt_u; + +/**************************************************************************//** + @Description A union holding all known protocol fields +*//***************************************************************************/ +typedef union ioc_fm_pcd_fields_u { + ioc_header_field_eth_t eth; /**< Ethernet */ + ioc_header_field_vlan_t vlan; /**< VLAN */ + ioc_header_field_llc_snap_t llc_snap; /**< LLC SNAP */ + ioc_header_field_pppoe_t pppoe; /**< PPPoE */ + ioc_header_field_mpls_t mpls; /**< MPLS */ + ioc_header_field_ip_t ip; /**< IP */ + ioc_header_field_ipv4_t ipv4; /**< IPv4 */ + ioc_header_field_ipv6_t ipv6; /**< IPv6 */ + ioc_header_field_udp_t udp; /**< UDP */ + ioc_header_field_udp_lite_t udp_lite; /**< UDP_Lite */ + ioc_header_field_tcp_t tcp; /**< TCP */ + ioc_header_field_sctp_t sctp; /**< SCTP */ + ioc_header_field_dccp_t dccp; /**< DCCP */ + ioc_header_field_gre_t gre; /**< GRE */ + ioc_header_field_minencap_t minencap; /**< Minimal Encapsulation */ + ioc_header_field_ipsec_ah_t ipsec_ah; /**< IPSec AH */ + ioc_header_field_ipsec_esp_t ipsec_esp; /**< IPSec ESP */ + ioc_header_field_udp_encap_esp_t udp_encap_esp; /**< UDP Encapsulation ESP */ +} ioc_fm_pcd_fields_u; + +/**************************************************************************//** + @Description Parameters for defining header extraction for key generation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_from_hdr_t { + uint8_t size; /**< Size in byte */ + uint8_t offset; /**< Byte offset */ +} ioc_fm_pcd_from_hdr_t; + +/**************************************************************************//** + @Description Parameters for defining field extraction for key generation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_from_field_t { + ioc_fm_pcd_fields_u field; /**< Field selection */ + uint8_t size; /**< Size in byte */ + uint8_t offset; /**< Byte offset */ +} ioc_fm_pcd_from_field_t; + +/**************************************************************************//** + @Description Parameters for defining a single network environment unit + A distinction unit should be defined if it will later be used + by one or more PCD engines to distinguish between flows. + (Must match struct t_FmPcdDistinctionUnit defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_distinction_unit_t { + struct { + ioc_net_header_type hdr; /**< One of the headers supported by the FM */ + ioc_fm_pcd_hdr_protocol_opt_u opt; /**< Select only one option! */ + } hdrs[IOC_FM_PCD_MAX_NUM_OF_INTERCHANGEABLE_HDRS]; +} ioc_fm_pcd_distinction_unit_t; + +/**************************************************************************//** + @Description Parameters for defining all different distinction units supported + by a specific PCD Network Environment Characteristics module. + + Each unit represent a protocol or a group of protocols that may + be used later by the different PCD engines to distinguish between flows. + (Must match struct t_FmPcdNetEnvParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_net_env_params_t { + uint8_t num_of_distinction_units;/**< Number of different units to be identified */ + ioc_fm_pcd_distinction_unit_t units[IOC_FM_PCD_MAX_NUM_OF_DISTINCTION_UNITS]; + /**< An array of num_of_distinction_units of the + different units to be identified */ + void *id; /**< Output parameter; Returns the net-env Id to be used */ +} ioc_fm_pcd_net_env_params_t; + +/**************************************************************************//** + @Description Parameters for defining a single extraction action when + creating a key +*//***************************************************************************/ +typedef struct ioc_fm_pcd_extract_entry_t { + ioc_fm_pcd_extract_type type; /**< Extraction type select */ + union { + struct { + ioc_net_header_type hdr; /**< Header selection */ + bool ignore_protocol_validation; + /**< Ignore protocol validation */ + ioc_fm_pcd_hdr_index hdr_index; /**< Relevant only for MPLS, VLAN and tunneled + IP. Otherwise should be cleared.*/ + ioc_fm_pcd_extract_by_hdr_type type; /**< Header extraction type select */ + union { + ioc_fm_pcd_from_hdr_t from_hdr; /**< Extract bytes from header parameters */ + ioc_fm_pcd_from_field_t from_field; /**< Extract bytes from field parameters */ + ioc_fm_pcd_fields_u full_field; /**< Extract full field parameters */ + } extract_by_hdr_type; + } extract_by_hdr; /**< Used when type = e_IOC_FM_PCD_KG_EXTRACT_BY_HDR */ + struct { + ioc_fm_pcd_extract_from src; /**< Non-header extraction source */ + ioc_fm_pcd_action action; /**< Relevant for CC Only */ + uint16_t ic_indx_mask; /**< Relevant only for CC when + action = e_IOC_FM_PCD_ACTION_INDEXED_LOOKUP; + Note that the number of bits that are set within + this mask must be log2 of the CC-node 'num_of_keys'. + Note that the mask cannot be set on the lower bits. */ + uint8_t offset; /**< Byte offset */ + uint8_t size; /**< Size in bytes */ + } extract_non_hdr; /**< Used when type = e_IOC_FM_PCD_KG_EXTRACT_NON_HDR */ + } extract_params; +} ioc_fm_pcd_extract_entry_t; + +/**************************************************************************//** + @Description A structure for defining masks for each extracted + field in the key. +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_extract_mask_t { + uint8_t extract_array_index; /**< Index in the extraction array, as initialized by user */ + uint8_t offset; /**< Byte offset */ + uint8_t mask; /**< A byte mask (selected bits will be ignored) */ +} ioc_fm_pcd_kg_extract_mask_t; + +/**************************************************************************//** + @Description A structure for defining default selection per groups + of fields +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_extract_dflt_t { + ioc_fm_pcd_kg_known_fields_dflt_types type; /**< Default type select*/ + ioc_fm_pcd_kg_extract_dflt_select dflt_select; /**< Default register select */ +} ioc_fm_pcd_kg_extract_dflt_t; + + +/**************************************************************************//** + @Description A structure for defining all parameters needed for + generation a key and using a hash function +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_key_extract_and_hash_params_t { + uint32_t private_dflt0; /**< Scheme default register 0 */ + uint32_t private_dflt1; /**< Scheme default register 1 */ + uint8_t num_of_used_extracts; /**< defines the valid size of the following array */ + ioc_fm_pcd_extract_entry_t extract_array [IOC_FM_PCD_KG_MAX_NUM_OF_EXTRACTS_PER_KEY]; + /**< An array of extraction definitions. */ + uint8_t num_of_used_dflts; /**< defines the valid size of the following array */ + ioc_fm_pcd_kg_extract_dflt_t dflts[IOC_FM_PCD_KG_NUM_OF_DEFAULT_GROUPS]; + /**< For each extraction used in this scheme, specify the required + default register to be used when header is not found. + types not specified in this array will get undefined value. */ + uint8_t num_of_used_masks; /**< Defines the valid size of the following array */ + ioc_fm_pcd_kg_extract_mask_t masks[IOC_FM_PCD_KG_NUM_OF_EXTRACT_MASKS]; + uint8_t hash_shift; /**< Hash result right shift. + Selects the 24 bits out of the 64 hash result. + 0 means using the 24 LSB's, otherwise use the + 24 LSB's after shifting right.*/ + uint32_t hash_distribution_num_of_fqids; /**< must be > 1 and a power of 2. Represents the range + of queues for the key and hash functionality */ + uint8_t hash_distribution_fqids_shift; /**< selects the FQID bits that will be effected by the hash */ + bool symmetric_hash; /**< TRUE to generate the same hash for frames with swapped source and + destination fields on all layers; If TRUE, driver will check that for + all layers, if SRC extraction is selected, DST extraction must also be + selected, and vice versa. */ +} ioc_fm_pcd_kg_key_extract_and_hash_params_t; + +/**************************************************************************//** + @Description A structure of parameters for defining a single + Qid mask (extracted OR). +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_extracted_or_params_t { + ioc_fm_pcd_extract_type type; /**< Extraction type select */ + union { + struct { /**< used when type = e_IOC_FM_PCD_KG_EXTRACT_BY_HDR */ + ioc_net_header_type hdr; + ioc_fm_pcd_hdr_index hdr_index; /**< Relevant only for MPLS, VLAN and tunneled + IP. Otherwise should be cleared.*/ + bool ignore_protocol_validation; + + } extract_by_hdr; + ioc_fm_pcd_extract_from src; /**< used when type = e_IOC_FM_PCD_KG_EXTRACT_NON_HDR */ + } extract_params; + uint8_t extraction_offset; /**< Offset for extraction */ + ioc_fm_pcd_kg_extract_dflt_select dflt_value; /**< Select register from which extraction is taken if + field not found */ + uint8_t mask; /**< Mask LSB byte of extraction (specified bits are ignored) */ + uint8_t bit_offset_in_fqid; /**< 0-31, Selects which bits of the 24 FQID bits to effect using + the extracted byte; Assume byte is placed as the 8 MSB's in + a 32 bit word where the lower bits + are the FQID; i.e if bitOffsetInFqid=1 than its LSB + will effect the FQID MSB, if bitOffsetInFqid=24 than the + extracted byte will effect the 8 LSB's of the FQID, + if bitOffsetInFqid=31 than the byte's MSB will effect + the FQID's LSB; 0 means - no effect on FQID; + Note that one, and only one of + bitOffsetInFqid or bitOffsetInPlcrProfile must be set (i.e, + extracted byte must effect either FQID or Policer profile).*/ + uint8_t bit_offset_in_plcr_profile; + /**< 0-15, Selects which bits of the 8 policer profile id bits to + effect using the extracted byte; Assume byte is placed + as the 8 MSB's in a 16 bit word where the lower bits + are the policer profile id; i.e if bitOffsetInPlcrProfile=1 + than its LSB will effect the profile MSB, if bitOffsetInFqid=8 + than the extracted byte will effect the whole policer profile id, + if bitOffsetInFqid=15 than the byte's MSB will effect + the Policer Profile id's LSB; + 0 means - no effect on policer profile; Note that one, and only one of + bitOffsetInFqid or bitOffsetInPlcrProfile must be set (i.e, + extracted byte must effect either FQID or Policer profile).*/ +} ioc_fm_pcd_kg_extracted_or_params_t; + +/**************************************************************************//** + @Description A structure for configuring scheme counter +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_scheme_counter_t { + bool update; /**< FALSE to keep the current counter state + and continue from that point, TRUE to update/reset + the counter when the scheme is written. */ + uint32_t value; /**< If update=TRUE, this value will be written into the + counter; clear this field to reset the counter. */ +} ioc_fm_pcd_kg_scheme_counter_t; + + +/**************************************************************************//** + @Description A structure for retrieving FMKG_SE_SPC +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_scheme_spc_t { + uint32_t val; /**< return value */ + void *id; /**< scheme handle */ +} ioc_fm_pcd_kg_scheme_spc_t; + +/**************************************************************************//** + @Description A structure for defining policer profile parameters as required by keygen + (when policer is the next engine after this scheme). + (Must match struct t_FmPcdKgPlcrProfile defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_plcr_profile_t { + bool shared_profile; /**< TRUE if this profile is shared between ports + (i.e. managed by master partition) May not be TRUE + if profile is after Coarse Classification*/ + bool direct; /**< If TRUE, direct_relative_profile_id only selects the profile + id, if FALSE fqid_offset_relative_profile_id_base is used + together with fqid_offset_shift and num_of_profiles + parameters, to define a range of profiles from + which the KeyGen result will determine the + destination policer profile. */ + union { + uint16_t direct_relative_profile_id; /**< Used if 'direct' is TRUE, to select policer profile. + This parameter should indicate the policer profile offset within the port's + policer profiles or SHARED window. */ + struct { + uint8_t fqid_offset_shift; /**< Shift of KG results without the qid base */ + uint8_t fqid_offset_relative_profile_id_base; + /**< OR of KG results without the qid base + This parameter should indicate the policer profile + offset within the port's policer profiles window + or SHARED window depends on shared_profile */ + uint8_t num_of_profiles; /**< Range of profiles starting at base */ + } indirect_profile; /**< Indirect profile parameters */ + } profile_select; /**< Direct/indirect profile selection and parameters */ +} ioc_fm_pcd_kg_plcr_profile_t; + +/**************************************************************************//** + @Description Parameters for configuring a storage profile for a KeyGen scheme. +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_storage_profile_t { + bool direct; /**< If TRUE, directRelativeProfileId only selects the + profile id; + If FALSE, fqidOffsetRelativeProfileIdBase is used + together with fqidOffsetShift and numOfProfiles + parameters to define a range of profiles from which + the KeyGen result will determine the destination + storage profile. */ + union { + uint16_t direct_relative_profileId; /**< Used when 'direct' is TRUE, to select a storage profile; + should indicate the storage profile offset within the + port's storage profiles window. */ + struct { + uint8_t fqid_offset_shift; /**< Shift of KeyGen results without the FQID base */ + uint8_t fqid_offset_relative_profile_id_base; + /**< OR of KeyGen results without the FQID base; + should indicate the policer profile offset within the + port's storage profiles window. */ + uint8_t num_of_profiles; /**< Range of profiles starting at base. */ + } indirect_profile; /**< Indirect profile parameters. */ + } profile_select; /**< Direct/indirect profile selection and parameters. */ +} ioc_fm_pcd_kg_storage_profile_t; + +/**************************************************************************//** + @Description Parameters for defining CC as the next engine after KeyGen + (Must match struct t_FmPcdKgCc defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_cc_t { + void *tree_id; /**< CC Tree id */ + uint8_t grp_id; /**< CC group id within the CC tree */ + bool plcr_next; /**< TRUE if after CC, in case of data frame, + policing is required. */ + bool bypass_plcr_profile_generation; + /**< TRUE to bypass KeyGen policer profile generation; + selected profile is the one set at port initialization. */ + ioc_fm_pcd_kg_plcr_profile_t plcr_profile; /**< Valid only if plcr_next = TRUE and + bypass_plcr_profile_generation = FALSE */ +} ioc_fm_pcd_kg_cc_t; + +/**************************************************************************//** + @Description Parameters for defining initializing a KeyGen scheme + (Must match struct t_FmPcdKgSchemeParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_kg_scheme_params_t { + bool modify; /**< TRUE to change an existing scheme */ + union { + uint8_t relative_scheme_id; + /**< if modify=FALSE: partition-relative scheme id */ + void *scheme_id; /**< if modify=TRUE: the id of an existing scheme */ + } scm_id; + bool shared; /**< This scheme is shared between ports + (must match t_FmPcdKgSchemeParams layout for memcpy in fmlib) */ + bool always_direct; /**< This scheme is reached only directly, i.e. no need + for match vector; KeyGen will ignore it when matching */ + struct { /**< HL relevant only if always_direct=FALSE */ + void *net_env_id; /**< The id of the Network Environment as returned + by FM_PCD_NetEnvCharacteristicsSet() */ + uint8_t num_of_distinction_units; + /**< Number of NetEnv units listed in unit_ids array */ + uint8_t unit_ids[IOC_FM_PCD_MAX_NUM_OF_DISTINCTION_UNITS]; + /**< Indexes as passed to SetNetEnvCharacteristics (?) array */ + } net_env_params; + bool use_hash; /**< use the KG Hash functionality */ + ioc_fm_pcd_kg_key_extract_and_hash_params_t key_extract_and_hash_params; + /**< used only if useHash = TRUE */ + bool bypass_fqid_generation; + /**< Normally - FALSE, TRUE to avoid FQID update in the IC; + In such a case FQID after KG will be the default FQID + defined for the relevant port, or the FQID defined by CC + in cases where CC was the previous engine. */ + uint32_t base_fqid; /**< Base FQID; Relevant only if bypass_fqid_generation = FALSE; + If hash is used and an even distribution is expected + according to hash_distribution_num_of_fqids, base_fqid must be aligned to + hash_distribution_num_of_fqids. */ + uint8_t num_of_used_extracted_ors; + /**< Number of FQID masks listed in extracted_ors array*/ + ioc_fm_pcd_kg_extracted_or_params_t extracted_ors[IOC_FM_PCD_KG_NUM_OF_GENERIC_REGS]; + /**< IOC_FM_PCD_KG_NUM_OF_GENERIC_REGS + registers are shared between qid_masks + functionality and some of the extraction + actions; Normally only some will be used + for qid_mask. Driver will return error if + resource is full at initialization time. */ + bool override_storage_profile; + /**< TRUE if KeyGen override previously decided storage profile */ + ioc_fm_pcd_kg_storage_profile_t storage_profile;/**< Used when override_storage_profile=TRUE */ + ioc_fm_pcd_engine next_engine; /**< may be BMI, PLCR or CC */ + union { /**< depends on nextEngine */ + ioc_fm_pcd_done_action done_action; /**< Used when next engine is BMI (done) */ + ioc_fm_pcd_kg_plcr_profile_t plcr_profile; /**< Used when next engine is PLCR */ + ioc_fm_pcd_kg_cc_t cc; /**< Used when next engine is CC */ + } kg_next_engine_params; + ioc_fm_pcd_kg_scheme_counter_t scheme_counter; /**< A structure of parameters for updating + the scheme counter */ + void *id; /**< Returns the scheme Id to be used */ +} ioc_fm_pcd_kg_scheme_params_t; + +/**************************************************************************//** + @Collection +*//***************************************************************************/ +#define IOC_FM_PCD_CC_STATS_MAX_NUM_OF_FLR 10 /* Maximal supported number of frame length ranges */ +#define IOC_FM_PCD_CC_STATS_FLR_SIZE 2 /* Size in bytes of a frame length range limit */ +#define IOC_FM_PCD_CC_STATS_FLR_COUNT_SIZE 4 /* Size in bytes of a frame length range counter */ +/* @} */ + +/**************************************************************************//** + @Description Parameters for defining CC as the next engine after a CC node. + (Must match struct t_FmPcdCcNextCcParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_next_cc_params_t { + void *cc_node_id; /**< Id of the next CC node */ +} ioc_fm_pcd_cc_next_cc_params_t; + +/**************************************************************************//** + @Description A structure for defining Frame Replicator as the next engine after a CC node. + (Must match struct t_FmPcdCcNextFrParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_next_fr_params_t { + void* frm_replic_id; /**< The id of the next frame replicator group */ +} ioc_fm_pcd_cc_next_fr_params_t; + +/**************************************************************************//** + @Description A structure for defining PLCR params when PLCR is the + next engine after a CC node + (Must match struct t_FmPcdCcNextPlcrParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_next_plcr_params_t { + bool override_params; /**< TRUE if CC override previously decided parameters*/ + bool shared_profile; /**< Relevant only if overrideParams=TRUE: + TRUE if this profile is shared between ports */ + uint16_t new_relative_profile_id; /**< Relevant only if overrideParams=TRUE: + (otherwise profile id is taken from keygen); + This parameter should indicate the policer + profile offset within the port's + policer profiles or from SHARED window.*/ + uint32_t new_fqid; /**< Relevant only if overrideParams=TRUE: + FQID for enquing the frame; + In earlier chips if policer next engine is KEYGEN, + this parameter can be 0, because the KEYGEN always decides + the enqueue FQID.*/ + uint8_t new_relative_storage_profile_id; + /**< Indicates the relative storage profile offset within + the port's storage profiles window; + Relevant only if the port was configured with VSP. */ +} ioc_fm_pcd_cc_next_plcr_params_t; + +/**************************************************************************//** + @Description A structure for defining enqueue params when BMI is the + next engine after a CC node + (Must match struct t_FmPcdCcNextEnqueueParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_next_enqueue_params_t { + ioc_fm_pcd_done_action action; /**< Action - when next engine is BMI (done) */ + bool override_fqid; /**< TRUE if CC override previously decided fqid and vspid, + relevant if action = e_IOC_FM_PCD_ENQ_FRAME */ + uint32_t new_fqid; /**< Valid if overrideFqid=TRUE, FQID for enqueuing the frame + (otherwise FQID is taken from KeyGen), + relevant if action = e_IOC_FM_PCD_ENQ_FRAME*/ + uint8_t new_relative_storage_profile_id; + /**< Valid if override_fqid=TRUE, Indicates the relative virtual + storage profile offset within the port's storage profiles + window; Relevant only if the port was configured with VSP. */ + +} ioc_fm_pcd_cc_next_enqueue_params_t; + +/**************************************************************************//** + @Description A structure for defining KG params when KG is the next engine after a CC node + (Must match struct t_FmPcdCcNextKgParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_next_kg_params_t { + bool override_fqid; /**< TRUE if CC override previously decided fqid and vspid, + Note - this parameters are irrelevant for earlier chips */ + uint32_t new_fqid; /**< Valid if overrideFqid=TRUE, FQID for enqueuing the frame + (otherwise FQID is taken from KeyGen), + Note - this parameters are irrelevant for earlier chips */ + uint8_t new_relative_storage_profile_id; + /**< Valid if override_fqid=TRUE, Indicates the relative virtual + storage profile offset within the port's storage profiles + window; Relevant only if the port was configured with VSP. */ + void *p_direct_scheme; /**< Direct scheme id to go to. */ +} ioc_fm_pcd_cc_next_kg_params_t; + +/**************************************************************************//** + @Description Parameters for defining the next engine after a CC node. + (Must match struct t_FmPcdCcNextEngineParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_next_engine_params_t { + ioc_fm_pcd_engine next_engine; /**< User has to initialize parameters + according to nextEngine definition */ + union { + ioc_fm_pcd_cc_next_cc_params_t cc_params; /**< Parameters in case next engine is CC */ + ioc_fm_pcd_cc_next_plcr_params_t plcr_params; /**< Parameters in case next engine is PLCR */ + ioc_fm_pcd_cc_next_enqueue_params_t enqueue_params; /**< Parameters in case next engine is BMI */ + ioc_fm_pcd_cc_next_kg_params_t kg_params; /**< Parameters in case next engine is KG */ + ioc_fm_pcd_cc_next_fr_params_t fr_params; /**< Parameters in case next engine is FR */ + } params; /**< Union used for all the next-engine parameters options */ + void *manip_id; /**< Handle to Manipulation object. + Relevant if next engine is of type result + (e_IOC_FM_PCD_PLCR, e_IOC_FM_PCD_KG, e_IOC_FM_PCD_DONE) */ + bool statistics_en; /**< If TRUE, statistics counters are incremented + for each frame passing through this + Coarse Classification entry. */ +} ioc_fm_pcd_cc_next_engine_params_t; + +/**************************************************************************//** + @Description Parameters for defining a single CC key +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_key_params_t { + uint8_t *p_key; /**< pointer to the key of the size defined in key_size */ + uint8_t *p_mask; /**< pointer to the Mask per key of the size defined + in keySize. p_key and p_mask (if defined) has to be + of the same size defined in the key_size */ + ioc_fm_pcd_cc_next_engine_params_t cc_next_engine_params; + /**< parameters for the next for the defined Key in p_key */ +#if 0 // following fields not defined in ioc_fm_pcd_cc_key_params_t +#if (DPAA_VERSION >= 11) + uint32_t internal_tstamp:1; /* set to use internal FMAN time stamp */ + uint32_t reserved:29; + uint32_t ext_timer_id:2; /* time stamp timer to use */ + uintptr_t monitor_addr; +#endif /* (DPAA_VERSION >= 11) */ +#endif // 0 + +} ioc_fm_pcd_cc_key_params_t; + +/**************************************************************************//** + @Description Parameters for defining CC keys parameters + The driver supports two methods for CC node allocation: dynamic and static. + Static mode was created in order to prevent runtime alloc/free + of FMan memory (MURAM), which may cause fragmentation; in this mode, + the driver automatically allocates the memory according to + 'max_num_of_keys' parameter. The driver calculates the maximal memory + size that may be used for this CC-Node taking into consideration + 'mask_support' and 'statistics_mode' parameters. + When 'action' = e_IOC_FM_PCD_ACTION_INDEXED_LOOKUP in the extraction + parameters of this node, 'max_num_of_keys' must be equal to 'num_of_keys'. + In dynamic mode, 'max_num_of_keys' must be zero. At initialization, + all required structures are allocated according to 'num_of_keys' + parameter. During runtime modification, these structures are + re-allocated according to the updated number of keys. + + Please note that 'action' and 'ic_indx_mask' mentioned in the + specific parameter explanations are passed in the extraction + parameters of the node (fields of extractccparams.extractnonhdr). +*//***************************************************************************/ +typedef struct ioc_keys_params_t { + uint16_t max_num_of_keys;/**< Maximum number of keys that will (ever) be used in this CC-Node; + A value of zero may be used for dynamic memory allocation. */ + bool mask_support; /**< This parameter is relevant only if a node is initialized with + action = e_IOC_FM_PCD_ACTION_EXACT_MATCH and max_num_of_keys > 0; + Should be TRUE to reserve table memory for key masks, even if + initial keys do not contain masks, or if the node was initialized + as 'empty' (without keys); this will allow user to add keys with + masks at runtime. */ + ioc_fm_pcd_cc_stats_mode statistics_mode;/**< Determines the supported statistics mode for all node's keys. + To enable statistics gathering, statistics should be enabled per + every key, using 'statistics_en' in next engine parameters structure + of that key; + If 'max_num_of_keys' is set, all required structures will be + preallocated for all keys. */ + uint16_t frame_length_ranges[IOC_FM_PCD_CC_STATS_MAX_NUM_OF_FLR]; + /**< Relevant only for 'RMON' statistics mode + (this feature is supported only on B4860 device); + Holds a list of programmable thresholds. For each received frame, + its length in bytes is examined against these range thresholds and + the appropriate counter is incremented by 1. For example, to belong + to range i, the following should hold: + range i-1 threshold < frame length <= range i threshold + Each range threshold must be larger then its preceding range + threshold. Last range threshold must be 0xFFFF. */ + uint16_t num_of_keys; /**< Number of initial keys; + Note that in case of 'action' = e_IOC_FM_PCD_ACTION_INDEXED_LOOKUP, + this field should be power-of-2 of the number of bits that are + set in 'ic_indx_mask'. */ + uint8_t key_size; /**< Size of key - for extraction of type FULL_FIELD, 'key_size' has + to be the standard size of the selected key; For other extraction + types, 'key_size' has to be as size of extraction; When 'action' = + e_IOC_FM_PCD_ACTION_INDEXED_LOOKUP, 'keySize' must be 2. */ + ioc_fm_pcd_cc_key_params_t key_params[IOC_FM_PCD_MAX_NUM_OF_KEYS]; + /**< An array with 'num_of_keys' entries, each entry specifies the + corresponding key parameters; + When 'action' = e_IOC_FM_PCD_ACTION_EXACT_MATCH, this value must not + exceed 255 (IOC_FM_PCD_MAX_NUM_OF_KEYS-1) as the last entry is saved + for the 'miss' entry. */ + ioc_fm_pcd_cc_next_engine_params_t cc_next_engine_params_for_miss; + /**< Parameters for defining the next engine when a key is not matched; + Not relevant if action = e_IOC_FM_PCD_ACTION_INDEXED_LOOKUP. */ +} ioc_keys_params_t; + +/**************************************************************************//** + @Description Parameters for defining a CC node +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_node_params_t { + ioc_fm_pcd_extract_entry_t extract_cc_params; /**< Extraction parameters */ + ioc_keys_params_t keys_params; /**< Keys definition matching the selected extraction */ + void *id; /**< Output parameter; returns the CC node Id to be used */ +} ioc_fm_pcd_cc_node_params_t; + +/**************************************************************************//** + @Description Parameters for defining a hash table + (Must match struct t_FmPcdHashTableParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_hash_table_params_t { + uint16_t max_num_of_keys; /**< Maximum Number Of Keys that will (ever) be used in this Hash-table */ + ioc_fm_pcd_cc_stats_mode statistics_mode; /**< If not e_IOC_FM_PCD_CC_STATS_MODE_NONE, the required structures for the + requested statistics mode will be allocated according to max_num_of_keys. */ + uint8_t kg_hash_shift; /**< KG-Hash-shift as it was configured in the KG-scheme + that leads to this hash-table. */ + uint16_t hash_res_mask; /**< Mask that will be used on the hash-result; + The number-of-sets for this hash will be calculated + as (2^(number of bits set in 'hash_res_mask')); + The 4 lower bits must be cleared. */ + uint8_t hash_shift; /**< Byte offset from the beginning of the KeyGen hash result to the + 2-bytes to be used as hash index. */ + uint8_t match_key_size; /**< Size of the exact match keys held by the hash buckets */ + + ioc_fm_pcd_cc_next_engine_params_t cc_next_engine_params_for_miss; + /**< Parameters for defining the next engine when a key is not matched */ + + bool aging_support; /**< TRUE to enable aging support for all keys of this hash table; + Aging status of a key enables the application to monitor if the + key was accessed for a certain period of time, meaning if a + packet that matches this key was received since this bit was last + set by the application */ + +#if (DPAA_VERSION >= 11) + bool external_hash; +#ifndef EXCLUDE_FMAN_IPR_OFFLOAD + uint32_t table_type; /* ip reassembly table */ + //valid for reassembly tables only + struct { + uint32_t timeout_val; //reassembly timeout + uint32_t timeout_fqid; //fqid for reassmebly failures + uint32_t max_frags; //max allowed fragments + uint32_t min_frag_size; //min allowed frag size except last frag + uint32_t max_sessions; //max conn reassembly sessions + }; +#endif + + struct { + uint8_t data_mem_id; /**< Memory partition ID for the external hash table buckets and contexts; + Must not cross the 4GB boundaries*/ + + uint16_t data_liodn_offs; /**< LIODN offset for access the external hash table buckets and contexts */ + + uintptr_t miss_monitor_addr; /**< user allocated miss monitor address */ + } external_hash_params; +#endif /* (DPAA_VERSION >= 11) */ + + void *id; +} ioc_fm_pcd_hash_table_params_t; + +/**************************************************************************//** + @Description A structure with the arguments for the FM_PCD_HashTableAddKey ioctl() call +*//***************************************************************************/ +typedef struct ioc_fm_pcd_hash_table_add_key_params_t { + void *p_hash_tbl; + uint8_t key_size; + ioc_fm_pcd_cc_key_params_t key_params; +} ioc_fm_pcd_hash_table_add_key_params_t; + +/**************************************************************************//** + @Description Parameters for defining a CC tree group. + + This structure defines a CC group in terms of NetEnv units + and the action to be taken in each case. The unit_ids list must + be given in order from low to high indices. + + ioc_fm_pcd_cc_next_engine_params_t is a list of 2^num_of_distinction_units + structures where each defines the next action to be taken for + each units combination. for example: + num_of_distinction_units = 2 + unit_ids = {1,3} + next_engine_per_entries_in_grp[0] = ioc_fm_pcd_cc_next_engine_params_t for the case that + unit 1 - not found; unit 3 - not found; + next_engine_per_entries_in_grp[1] = ioc_fm_pcd_cc_next_engine_params_t for the case that + unit 1 - not found; unit 3 - found; + next_engine_per_entries_in_grp[2] = ioc_fm_pcd_cc_next_engine_params_t for the case that + unit 1 - found; unit 3 - not found; + next_engine_per_entries_in_grp[3] = ioc_fm_pcd_cc_next_engine_params_t for the case that + unit 1 - found; unit 3 - found; +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_grp_params_t { + uint8_t num_of_distinction_units; /**< Up to 4 */ + uint8_t unit_ids [IOC_FM_PCD_MAX_NUM_OF_CC_UNITS]; + /**< Indexes of the units as defined in + FM_PCD_NetEnvCharacteristicsSet() */ + ioc_fm_pcd_cc_next_engine_params_t next_engine_per_entries_in_grp[IOC_FM_PCD_MAX_NUM_OF_CC_ENTRIES_IN_GRP]; + /**< Maximum entries per group is 16 */ +} ioc_fm_pcd_cc_grp_params_t; + +/**************************************************************************//** + @Description Parameters for defining the CC tree groups + (Must match struct t_FmPcdCcTreeParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_tree_params_t { + void *net_env_id; /**< Id of the Network Environment as returned + by FM_PCD_NetEnvCharacteristicsSet() */ + uint8_t num_of_groups; /**< Number of CC groups within the CC tree */ + ioc_fm_pcd_cc_grp_params_t fm_pcd_cc_group_params [IOC_FM_PCD_MAX_NUM_OF_CC_GROUPS]; + /**< Parameters for each group. */ + void *id; /**< Output parameter; Returns the tree Id to be used */ +} ioc_fm_pcd_cc_tree_params_t; + +/**************************************************************************//** + @Description Parameters for defining policer byte rate +*//***************************************************************************/ +typedef struct ioc_fm_pcd_plcr_byte_rate_mode_param_t { + ioc_fm_pcd_plcr_frame_length_select frame_length_selection; /**< Frame length selection */ + ioc_fm_pcd_plcr_roll_back_frame_select roll_back_frame_selection; /**< relevant option only e_IOC_FM_PCD_PLCR_L2_FRM_LEN, + e_IOC_FM_PCD_PLCR_FULL_FRM_LEN */ +} ioc_fm_pcd_plcr_byte_rate_mode_param_t; + +/**************************************************************************//** + @Description Parameters for defining the policer profile (based on + RFC-2698 or RFC-4115 attributes). +*//***************************************************************************/ +typedef struct ioc_fm_pcd_plcr_non_passthrough_alg_param_t { + ioc_fm_pcd_plcr_rate_mode rate_mode; /**< Byte / Packet */ + ioc_fm_pcd_plcr_byte_rate_mode_param_t byte_mode_param; /**< Valid for Byte NULL for Packet */ + uint32_t committed_info_rate; /**< KBits/Sec or Packets/Sec */ + uint32_t committed_burst_size; /**< KBits or Packets */ + uint32_t peak_or_excess_info_rate; /**< KBits/Sec or Packets/Sec */ + uint32_t peak_or_excess_burst_size; /**< KBits or Packets */ +} ioc_fm_pcd_plcr_non_passthrough_alg_param_t; + +/**************************************************************************//** + @Description Parameters for defining the next engine after policer +*//***************************************************************************/ +typedef union ioc_fm_pcd_plcr_next_engine_params_u { + ioc_fm_pcd_done_action action; /**< Action - when next engine is BMI (done) */ + void *p_profile; /**< Policer profile handle - used when next engine + is PLCR, must be a SHARED profile */ + void *p_direct_scheme; /**< Direct scheme select - when next engine is Keygen */ +} ioc_fm_pcd_plcr_next_engine_params_u; + +typedef struct ioc_fm_pcd_port_params_t { + ioc_fm_port_type port_type; /**< Type of port for this profile */ + uint8_t port_id; /**< FM-Port id of port for this profile */ +} ioc_fm_pcd_port_params_t; + +/**************************************************************************//** + @Description Parameters for defining the policer profile entry + (Must match struct t_FmPcdPlcrProfileParams defined in fm_pcd_ext.h) +*//***************************************************************************/ +typedef struct ioc_fm_pcd_plcr_profile_params_t { + bool modify; /**< TRUE to change an existing profile */ + union { + struct { + ioc_fm_pcd_profile_type_selection profile_type; /**< Type of policer profile */ + ioc_fm_pcd_port_params_t *p_fm_port; /**< Relevant for per-port profiles only */ + uint16_t relative_profile_id; /**< Profile id - relative to shared group or to port */ + } new_params; /**< Use it when modify = FALSE */ + void *p_profile; /**< A handle to a profile - use it when modify=TRUE */ + } profile_select; + ioc_fm_pcd_plcr_algorithm_selection alg_selection; /**< Profile Algorithm PASS_THROUGH, RFC_2698, RFC_4115 */ + ioc_fm_pcd_plcr_color_mode color_mode; /**< COLOR_BLIND, COLOR_AWARE */ + + union { + ioc_fm_pcd_plcr_color dflt_color; /**< For Color-Blind Pass-Through mode; the policer will re-color + any incoming packet with the default value. */ + ioc_fm_pcd_plcr_color override; /**< For Color-Aware modes; the profile response to a + pre-color value of 2'b11. */ + } color; + + ioc_fm_pcd_plcr_non_passthrough_alg_param_t non_passthrough_alg_param; /**< RFC2698 or RFC4115 parameters */ + + ioc_fm_pcd_engine next_engine_on_green; /**< Next engine for green-colored frames */ + ioc_fm_pcd_plcr_next_engine_params_u params_on_green; /**< Next engine parameters for green-colored frames */ + + ioc_fm_pcd_engine next_engine_on_yellow; /**< Next engine for yellow-colored frames */ + ioc_fm_pcd_plcr_next_engine_params_u params_on_yellow; /**< Next engine parameters for yellow-colored frames */ + + ioc_fm_pcd_engine next_engine_on_red; /**< Next engine for red-colored frames */ + ioc_fm_pcd_plcr_next_engine_params_u params_on_red; /**< Next engine parameters for red-colored frames */ + + bool trap_profile_on_flow_A; /**< Obsolete - do not use */ + bool trap_profile_on_flow_B; /**< Obsolete - do not use */ + bool trap_profile_on_flow_C; /**< Obsolete - do not use */ + + void *id; /**< output parameter; Returns the profile Id to be used */ +} ioc_fm_pcd_plcr_profile_params_t; + +/**************************************************************************//** + @Description A structure for modifying CC tree next engine +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_tree_modify_next_engine_params_t { + void *id; /**< CC tree Id to be used */ + uint8_t grp_indx; /**< A Group index in the tree */ + uint8_t indx; /**< Entry index in the group defined by grp_index */ + ioc_fm_pcd_cc_next_engine_params_t cc_next_engine_params; + /**< Parameters for the next for the defined Key in the p_Key */ +} ioc_fm_pcd_cc_tree_modify_next_engine_params_t; + +/**************************************************************************//** + @Description A structure for modifying CC node next engine +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_node_modify_next_engine_params_t { + void *id; /**< CC node Id to be used */ + uint16_t key_indx; /**< Key index for Next Engine Params modifications; + NOTE: This parameter is IGNORED for miss-key! */ + uint8_t key_size; /**< Key size of added key */ + ioc_fm_pcd_cc_next_engine_params_t cc_next_engine_params; + /**< parameters for the next for the defined Key in the p_Key */ +} ioc_fm_pcd_cc_node_modify_next_engine_params_t; + +/**************************************************************************//** + @Description A structure for remove CC node key +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_node_remove_key_params_t { + void *id; /**< CC node Id to be used */ + uint16_t key_indx; /**< Key index for Next Engine Params modifications; + NOTE: This parameter is IGNORED for miss-key! */ +} ioc_fm_pcd_cc_node_remove_key_params_t; + +/**************************************************************************//** + @Description A structure for modifying CC node key and next engine +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_node_modify_key_and_next_engine_params_t { + void *id; /**< CC node Id to be used */ + uint16_t key_indx; /**< Key index for Next Engine Params modifications; + NOTE: This parameter is IGNORED for miss-key! */ + uint8_t key_size; /**< Key size of added key */ + ioc_fm_pcd_cc_key_params_t key_params; /**< it's array with numOfKeys entries each entry in + the array of the type ioc_fm_pcd_cc_key_params_t */ +} ioc_fm_pcd_cc_node_modify_key_and_next_engine_params_t; + +/**************************************************************************//** + @Description A structure for modifying CC node key +*//***************************************************************************/ +typedef struct ioc_fm_pcd_cc_node_modify_key_params_t { + void *id; /**< CC node Id to be used */ + uint16_t key_indx; /**< Key index for Next Engine Params modifications; + NOTE: This parameter is IGNORED for miss-key! */ + uint8_t key_size; /**< Key size of added key */ + uint8_t *p_key; /**< Pointer to the key of the size defined in key_size */ + uint8_t *p_mask; /**< Pointer to the Mask per key of the size defined + in keySize. p_Key and p_Mask (if defined) have to be + of the same size as defined in the key_size */ +} ioc_fm_pcd_cc_node_modify_key_params_t; + +/**************************************************************************//** + @Description A structure with the arguments for the FM_PCD_HashTableRemoveKey ioctl() call +*//***************************************************************************/ +typedef struct ioc_fm_pcd_hash_table_remove_key_params_t { + void *p_hash_tbl; /**< The id of the hash table */ + uint8_t key_size; /**< The size of the key to remove */ + uint8_t *p_key; /**< Pointer to the key to remove */ +} ioc_fm_pcd_hash_table_remove_key_params_t; + +/**************************************************************************//** + @Description Parameters for selecting a location for requested manipulation +*//***************************************************************************/ +typedef struct ioc_fm_manip_hdr_info_t { + ioc_net_header_type hdr; /**< Header selection */ + ioc_fm_pcd_hdr_index hdr_index; /**< Relevant only for MPLS, VLAN and tunneled IP. Otherwise should be cleared. */ + bool by_field; /**< TRUE if the location of manipulation is according to some field in the specific header*/ + ioc_fm_pcd_fields_u full_field; /**< Relevant only when by_field = TRUE: Extract field */ +} ioc_fm_manip_hdr_info_t; + +/**************************************************************************//** + @Description Parameters for defining header removal by header type +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_rmv_by_hdr_params_t { + ioc_fm_pcd_manip_hdr_rmv_by_hdr_type type; /**< Selection of header removal location */ + union { + ioc_fm_manip_hdr_info_t hdr_info; /**< Relevant when type = e_FM_PCD_MANIP_RMV_BY_HDR_FROM_START */ + ioc_fm_pcd_manip_hdr_rmv_specific_l2 specific_l2;/**< Relevant when type = e_IOC_FM_PCD_MANIP_BY_HDR_SPECIFIC_L2; + Defines which L2 headers to remove. */ + } u; +} ioc_fm_pcd_manip_hdr_rmv_by_hdr_params_t; + +/**************************************************************************//** + @Description Parameters for configuring IP fragmentation manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_frag_ip_params_t { + uint16_t size_for_fragmentation; /**< If length of the frame is greater than this value, + IP fragmentation will be executed.*/ + bool sg_bpid_en; /**< Enable a dedicated buffer pool id for the Scatter/Gather buffer allocation; + If disabled, the Scatter/Gather buffer will be allocated from the same pool as the + received frame's buffer. */ + uint8_t sg_bpid; /**< Scatter/Gather buffer pool id; + This parameter is relevant when 'sg_bpid_en=TRUE'; + Same LIODN number is used for these buffers as for the received frames buffers, so buffers + of this pool need to be allocated in the same memory area as the received buffers. + If the received buffers arrive from different sources, the Scatter/Gather BP id should be + mutual to all these sources. */ + ioc_fm_pcd_manip_dont_frag_action dont_frag_action; /**< Dont Fragment Action - If an IP packet is larger + than MTU and its DF bit is set, then this field will + determine the action to be taken.*/ +} ioc_fm_pcd_manip_frag_ip_params_t; + +/**************************************************************************//** + @Description Parameters for configuring IP reassembly manipulation. + + This is a common structure for both IPv4 and IPv6 reassembly + manipulation. For reassembly of both IPv4 and IPv6, make sure to + set the 'hdr' field in ioc_fm_pcd_manip_reassem_params_t to IOC_HEADER_TYPE_IPv6. +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_reassem_ip_params_t { + uint8_t relative_scheme_id[2]; /**< Partition relative scheme id: + relativeSchemeId[0] - Relative scheme ID for IPV4 Reassembly manipulation; + relativeSchemeId[1] - Relative scheme ID for IPV6 Reassembly manipulation; + NOTE: The following comment is relevant only for FMAN v2 devices: + Relative scheme ID for IPv4/IPv6 Reassembly manipulation must be smaller than + the user schemes id to ensure that the reassembly's schemes will be first match. + The remaining schemes, if defined, should have higher relative scheme ID. */ + uint32_t non_consistent_sp_fqid; /**< In case that other fragments of the frame corresponds to different storage + profile than the opening fragment (Non-Consistent-SP state) + then one of two possible scenarios occurs: + if 'nonConsistentSpFqid != 0', the reassembled frame will be enqueued to + this fqid, otherwise a 'Non Consistent SP' bit will be set in the FD[status].*/ + uint8_t data_mem_id; /**< Memory partition ID for the IPR's external tables structure */ + uint16_t data_liodn_offset; /**< LIODN offset for access the IPR's external tables structure. */ + uint16_t min_frag_size[2]; /**< Minimum fragment size: + minFragSize[0] - for ipv4, minFragSize[1] - for ipv6 */ + ioc_fm_pcd_manip_reassem_ways_number num_of_frames_per_hash_entry[2]; + /**< Number of frames per hash entry needed for reassembly process: + numOfFramesPerHashEntry[0] - for ipv4 (max value is e_IOC_FM_PCD_MANIP_EIGHT_WAYS_HASH); + numOfFramesPerHashEntry[1] - for ipv6 (max value is e_IOC_FM_PCD_MANIP_SIX_WAYS_HASH). */ + uint16_t max_num_frames_in_process;/**< Number of frames which can be processed by Reassembly in the same time; + Must be power of 2; + In the case numOfFramesPerHashEntry == e_IOC_FM_PCD_MANIP_FOUR_WAYS_HASH, + maxNumFramesInProcess has to be in the range of 4 - 512; + In the case numOfFramesPerHashEntry == e_IOC_FM_PCD_MANIP_EIGHT_WAYS_HASH, + maxNumFramesInProcess has to be in the range of 8 - 2048. */ + ioc_fm_pcd_manip_reassem_time_out_mode time_out_mode; /**< Expiration delay initialized by Reassembly process */ + uint32_t fqid_for_time_out_frames;/**< FQID in which time out frames will enqueue during Time Out Process */ + uint32_t timeout_threshold_for_reassm_process; + /**< Represents the time interval in microseconds which defines + if opened frame (at least one fragment was processed but not all the fragments)is found as too old*/ +} ioc_fm_pcd_manip_reassem_ip_params_t; + +/**************************************************************************//** + @Description Parameters for defining IPSEC manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_special_offload_ipsec_params_t { + bool decryption; /**< TRUE if being used in decryption direction; + FALSE if being used in encryption direction. */ + bool ecn_copy; /**< TRUE to copy the ECN bits from inner/outer to outer/inner + (direction depends on the 'decryption' field). */ + bool dscp_copy; /**< TRUE to copy the DSCP bits from inner/outer to outer/inner + (direction depends on the 'decryption' field). */ + bool variable_ip_hdr_len; /**< TRUE for supporting variable IP header length in decryption. */ + bool variable_ip_version; /**< TRUE for supporting both IP version on the same SA in encryption */ + uint8_t outer_ip_hdr_len; /**< If 'variable_ip_version == TRUE' than this field must be set to non-zero value; + It is specifies the length of the outer IP header that was configured in the + corresponding SA. */ + uint16_t arw_size; /**< if <> '0' then will perform ARW check for this SA; + The value must be a multiplication of 16 */ + void *arw_addr; /**< if arwSize <> '0' then this field must be set to non-zero value; + MUST be allocated from FMAN's MURAM that the post-sec op-port belong + Must be 4B aligned. Required MURAM size is '(NEXT_POWER_OF_2(arwSize+32))/8+4' Bytes */ +} ioc_fm_pcd_manip_special_offload_ipsec_params_t; + +/**************************************************************************//** + @Description Parameters for configuring CAPWAP fragmentation manipulation + + Restrictions: + - Maximum number of fragments per frame is 16. + - Transmit confirmation is not supported. + - Fragmentation nodes must be set as the last PCD action (i.e. the + corresponding CC node key must have next engine set to e_FM_PCD_DONE). + - Only BMan buffers shall be used for frames to be fragmented. + - NOTE: The following comment is relevant only for FMAN v3 devices: IPF + does not support VSP. Therefore, on the same port where we have IPF we + cannot support VSP. +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_frag_capwap_params_t { + uint16_t size_for_fragmentation; /**< If length of the frame is greater than this value, + CAPWAP fragmentation will be executed.*/ + bool sg_bpid_en; /**< Enable a dedicated buffer pool id for the Scatter/Gather buffer allocation; + If disabled, the Scatter/Gather buffer will be allocated from the same pool as the + received frame's buffer. */ + uint8_t sg_bpid; /**< Scatter/Gather buffer pool id; + This parameters is relevant when 'sgBpidEn=TRUE'; + Same LIODN number is used for these buffers as for the received frames buffers, so buffers + of this pool need to be allocated in the same memory area as the received buffers. + If the received buffers arrive from different sources, the Scatter/Gather BP id should be + mutual to all these sources. */ + bool compress_mode_en; /**< CAPWAP Header Options Compress Enable mode; + When this mode is enabled then only the first fragment include the CAPWAP header options + field (if user provides it in the input frame) and all other fragments exclude the CAPWAP + options field (CAPWAP header is updated accordingly).*/ +} ioc_fm_pcd_manip_frag_capwap_params_t; + +/**************************************************************************//** + @Description Parameters for configuring CAPWAP reassembly manipulation. + + Restrictions: + - Application must define one scheme to catch the reassembled frames. + - Maximum number of fragments per frame is 16. + +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_reassem_capwap_params_t { + uint8_t relative_scheme_id; /**< Partition relative scheme id; + NOTE: this id must be smaller than the user schemes id to ensure that the reassembly scheme will be first match; + Rest schemes, if defined, should have higher relative scheme ID. */ + uint8_t data_mem_id; /**< Memory partition ID for the IPR's external tables structure */ + uint16_t data_liodn_offset; /**< LIODN offset for access the IPR's external tables structure. */ + uint16_t max_reassembled_frame_length;/**< The maximum CAPWAP reassembled frame length in bytes; + If maxReassembledFrameLength == 0, any successful reassembled frame length is + considered as a valid length; + if maxReassembledFrameLength > 0, a successful reassembled frame which its length + exceeds this value is considered as an error frame (FD status[CRE] bit is set). */ + ioc_fm_pcd_manip_reassem_ways_number num_of_frames_per_hash_entry; + /**< Number of frames per hash entry needed for reassembly process */ + uint16_t max_num_frames_in_process; /**< Number of frames which can be processed by reassembly in the same time; + Must be power of 2; + In the case numOfFramesPerHashEntry == e_FM_PCD_MANIP_FOUR_WAYS_HASH, + maxNumFramesInProcess has to be in the range of 4 - 512; + In the case numOfFramesPerHashEntry == e_FM_PCD_MANIP_EIGHT_WAYS_HASH, + maxNumFramesInProcess has to be in the range of 8 - 2048. */ + ioc_fm_pcd_manip_reassem_time_out_mode time_out_mode; /**< Expiration delay initialized by Reassembly process */ + uint32_t fqid_for_time_out_frames; /**< FQID in which time out frames will enqueue during Time Out Process; + Recommended value for this field is 0; in this way timed-out frames will be discarded */ + uint32_t timeout_threshold_for_reassm_process; + /**< Represents the time interval in microseconds which defines + if opened frame (at least one fragment was processed but not all the fragments)is found as too old*/ +} ioc_fm_pcd_manip_reassem_capwap_params_t; + +/**************************************************************************//** + @Description structure for defining CAPWAP manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_special_offload_capwap_params_t { + bool dtls; /**< TRUE if continue to SEC DTLS encryption */ + ioc_fm_pcd_manip_hdr_qos_src qos_src; /**< TODO */ +} ioc_fm_pcd_manip_special_offload_capwap_params_t; + + +/**************************************************************************//** + @Description Parameters for defining special offload manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_special_offload_params_t { + ioc_fm_pcd_manip_special_offload_type type; /**< Type of special offload manipulation */ + union + { + ioc_fm_pcd_manip_special_offload_ipsec_params_t ipsec; /**< Parameters for IPSec; Relevant when + type = e_IOC_FM_PCD_MANIP_SPECIAL_OFFLOAD_IPSEC */ + + ioc_fm_pcd_manip_special_offload_capwap_params_t capwap; /**< Parameters for CAPWAP; Relevant when + type = e_FM_PCD_MANIP_SPECIAL_OFFLOAD_CAPWAP */ + } u; +} ioc_fm_pcd_manip_special_offload_params_t; + +/**************************************************************************//** + @Description Parameters for defining generic removal manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_rmv_generic_params_t { + uint8_t offset; /**< Offset from beginning of header to the start + location of the removal */ + uint8_t size; /**< Size of removed section */ +} ioc_fm_pcd_manip_hdr_rmv_generic_params_t; + +/**************************************************************************//** + @Description Parameters for defining insertion manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_insrt_t { + uint8_t size; /**< size of inserted section */ + uint8_t *p_data; /**< data to be inserted */ +} ioc_fm_pcd_manip_hdr_insrt_t; + +/**************************************************************************//** + @Description Parameters for defining generic insertion manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_insrt_generic_params_t { + uint8_t offset; /**< Offset from beginning of header to the start + location of the insertion */ + uint8_t size; /**< Size of inserted section */ + bool replace; /**< TRUE to override (replace) existing data at + 'offset', FALSE to insert */ + uint8_t *p_data; /**< Pointer to data to be inserted */ +} ioc_fm_pcd_manip_hdr_insrt_generic_params_t; + +/**************************************************************************//** + @Description Parameters for defining header manipulation VLAN DSCP To Vpri translation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_field_update_vlan_dscp_to_vpri_t { + uint8_t dscp_to_vpri_table[IOC_FM_PCD_MANIP_DSCP_TO_VLAN_TRANS]; + /**< A table of VPri values for each DSCP value; + The index is the D_SCP value (0-0x3F) and the + value is the corresponding VPRI (0-15). */ + uint8_t vpri_def_val; /**< 0-7, Relevant only if if update_type = + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_DSCP_TO_VLAN, + this field is the Q Tag default value if the + IP header is not found. */ +} ioc_fm_pcd_manip_hdr_field_update_vlan_dscp_to_vpri_t; + +/**************************************************************************//** + @Description Parameters for defining header manipulation VLAN fields updates +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_field_update_vlan_t { + ioc_fm_pcd_manip_hdr_field_update_vlan update_type; /**< Selects VLAN update type */ + union { + uint8_t vpri; /**< 0-7, Relevant only if If update_type = + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_VLAN_PRI, this + is the new VLAN pri. */ + ioc_fm_pcd_manip_hdr_field_update_vlan_dscp_to_vpri_t dscp_to_vpri; + /**< Parameters structure, Relevant only if update_type = + e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_DSCP_TO_VLAN. */ + } u; +} ioc_fm_pcd_manip_hdr_field_update_vlan_t; + +/**************************************************************************//** + @Description Parameters for defining header manipulation IPV4 fields updates +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_field_update_ipv4_t { + ioc_ipv4_hdr_manip_update_flags_t valid_updates; /**< ORed flag, selecting the required updates */ + uint8_t tos; /**< 8 bit New TOS; Relevant if valid_updates contains + IOC_HDR_MANIP_IPV4_TOS */ + uint16_t id; /**< 16 bit New IP ID; Relevant only if valid_updates + contains IOC_HDR_MANIP_IPV4_ID */ + uint32_t src; /**< 32 bit New IP SRC; Relevant only if valid_updates + contains IOC_HDR_MANIP_IPV4_SRC */ + uint32_t dst; /**< 32 bit New IP DST; Relevant only if valid_updates + contains IOC_HDR_MANIP_IPV4_DST */ +} ioc_fm_pcd_manip_hdr_field_update_ipv4_t; + +/**************************************************************************//** + @Description Parameters for defining header manipulation IPV6 fields updates +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_field_update_ipv6_t { + ioc_ipv6_hdr_manip_update_flags_t valid_updates; /**< ORed flag, selecting the required updates */ + uint8_t traffic_class; /**< 8 bit New Traffic Class; Relevant if valid_updates contains + IOC_HDR_MANIP_IPV6_TC */ + uint8_t src[IOC_NET_HEADER_FIELD_IPv6_ADDR_SIZE]; + /**< 16 byte new IP SRC; Relevant only if valid_updates + contains IOC_HDR_MANIP_IPV6_SRC */ + uint8_t dst[IOC_NET_HEADER_FIELD_IPv6_ADDR_SIZE]; + /**< 16 byte new IP DST; Relevant only if valid_updates + contains IOC_HDR_MANIP_IPV6_DST */ +} ioc_fm_pcd_manip_hdr_field_update_ipv6_t; + +/**************************************************************************//** + @Description Parameters for defining header manipulation TCP/UDP fields updates +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_field_update_tcp_udp_t { + ioc_tcp_udp_hdr_manip_update_flags_t valid_updates; /**< ORed flag, selecting the required updates */ + uint16_t src; /**< 16 bit New TCP/UDP SRC; Relevant only if valid_updates + contains IOC_HDR_MANIP_TCP_UDP_SRC */ + uint16_t dst; /**< 16 bit New TCP/UDP DST; Relevant only if valid_updates + contains IOC_HDR_MANIP_TCP_UDP_DST */ +} ioc_fm_pcd_manip_hdr_field_update_tcp_udp_t; + +/**************************************************************************//** + @Description Parameters for defining header manipulation fields updates +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_field_update_params_t { + ioc_fm_pcd_manip_hdr_field_update_type type; /**< Type of header field update manipulation */ + union { + ioc_fm_pcd_manip_hdr_field_update_vlan_t vlan; /**< Parameters for VLAN update. Relevant when + type = e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_VLAN */ + ioc_fm_pcd_manip_hdr_field_update_ipv4_t ipv4; /**< Parameters for IPv4 update. Relevant when + type = e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_IPV4 */ + ioc_fm_pcd_manip_hdr_field_update_ipv6_t ipv6; /**< Parameters for IPv6 update. Relevant when + type = e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_IPV6 */ + ioc_fm_pcd_manip_hdr_field_update_tcp_udp_t tcp_udp;/**< Parameters for TCP/UDP update. Relevant when + type = e_IOC_FM_PCD_MANIP_HDR_FIELD_UPDATE_TCP_UDP */ + } u; +} ioc_fm_pcd_manip_hdr_field_update_params_t; + +/**************************************************************************//** + @Description Parameters for defining custom header manipulation for IP replacement +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_custom_ip_hdr_replace_t { + ioc_fm_pcd_manip_hdr_custom_ip_replace replace_type; /**< Selects replace update type */ + bool dec_ttl_hl; /**< Decrement TTL (IPV4) or Hop limit (IPV6) by 1 */ + bool update_ipv4_id; /**< Relevant when replace_type = + e_IOC_FM_PCD_MANIP_HDR_CUSTOM_REPLACE_IPV6_BY_IPV4 */ + uint16_t id; /**< 16 bit New IP ID; Relevant only if + update_ipv4_id = TRUE */ + uint8_t hdr_size; /**< The size of the new IP header */ + uint8_t hdr[IOC_FM_PCD_MANIP_MAX_HDR_SIZE]; + /**< The new IP header */ +} ioc_fm_pcd_manip_hdr_custom_ip_hdr_replace_t; + +/**************************************************************************//** + @Description Parameters for defining custom header manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_custom_params_t { + ioc_fm_pcd_manip_hdr_custom_type type; /**< Type of header field update manipulation */ + union { + ioc_fm_pcd_manip_hdr_custom_ip_hdr_replace_t ip_hdr_replace; + /**< Parameters IP header replacement */ + } u; +} ioc_fm_pcd_manip_hdr_custom_params_t; + +/**************************************************************************//** + @Description Parameters for defining specific L2 insertion manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_insrt_specific_l2_params_t { + ioc_fm_pcd_manip_hdr_insrt_specific_l2 specific_l2; /**< Selects which L2 headers to insert */ + bool update; /**< TRUE to update MPLS header */ + uint8_t size; /**< size of inserted section */ + uint8_t *p_data; /**< data to be inserted */ +} ioc_fm_pcd_manip_hdr_insrt_specific_l2_params_t; + +/**************************************************************************//** + @Description Parameters for defining IP insertion manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_insrt_ip_params_t { + bool calc_l4_checksum; /**< Calculate L4 checksum. */ + ioc_fm_pcd_manip_hdr_qos_mapping_mode mapping_mode; /**< TODO */ + uint8_t last_pid_offset; /**< the offset of the last Protocol within + the inserted header */ + uint16_t id; /**< 16 bit New IP ID */ + bool dont_frag_overwrite; + /**< IPv4 only. DF is overwritten with the hash-result next-to-last byte. + * This byte is configured to be overwritten when RPD is set. */ + uint8_t last_dst_offset; + /**< IPv6 only. if routing extension exist, user should set the offset of the destination address + * in order to calculate UDP checksum pseudo header; + * Otherwise set it to '0'. */ + ioc_fm_pcd_manip_hdr_insrt_t insrt; /**< size and data to be inserted. */ +} ioc_fm_pcd_manip_hdr_insrt_ip_params_t; + +/**************************************************************************//** + @Description Parameters for defining header insertion manipulation by header type +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_insrt_by_hdr_params_t { + ioc_fm_pcd_manip_hdr_insrt_by_hdr_type type; /**< Selects manipulation type */ + union { + ioc_fm_pcd_manip_hdr_insrt_specific_l2_params_t specific_l2_params; + /**< Used when type = e_IOC_FM_PCD_MANIP_INSRT_BY_HDR_SPECIFIC_L2: + Selects which L2 headers to remove */ + ioc_fm_pcd_manip_hdr_insrt_ip_params_t ip_params; /**< Used when type = e_FM_PCD_MANIP_INSRT_BY_HDR_IP */ + ioc_fm_pcd_manip_hdr_insrt_t insrt; /**< Used when type is one of e_FM_PCD_MANIP_INSRT_BY_HDR_UDP, + e_FM_PCD_MANIP_INSRT_BY_HDR_UDP_LITE, or + e_FM_PCD_MANIP_INSRT_BY_HDR_CAPWAP */ + } u; +} ioc_fm_pcd_manip_hdr_insrt_by_hdr_params_t; + +/**************************************************************************//** + @Description Parameters for defining header insertion manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_insrt_params_t { + ioc_fm_pcd_manip_hdr_insrt_type type; /**< Type of insertion manipulation */ + union { + ioc_fm_pcd_manip_hdr_insrt_by_hdr_params_t by_hdr; /**< Parameters for defining header insertion manipulation by header type, + relevant if 'type' = e_IOC_FM_PCD_MANIP_INSRT_BY_HDR */ + ioc_fm_pcd_manip_hdr_insrt_generic_params_t generic;/**< Parameters for defining generic header insertion manipulation, + relevant if type = e_IOC_FM_PCD_MANIP_INSRT_GENERIC */ + } u; +} ioc_fm_pcd_manip_hdr_insrt_params_t; + +/**************************************************************************//** + @Description Parameters for defining header removal manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_rmv_params_t { + ioc_fm_pcd_manip_hdr_rmv_type type; /**< Type of header removal manipulation */ + union { + ioc_fm_pcd_manip_hdr_rmv_by_hdr_params_t by_hdr; /**< Parameters for defining header removal manipulation by header type, + relevant if type = e_IOC_FM_PCD_MANIP_RMV_BY_HDR */ + ioc_fm_pcd_manip_hdr_rmv_generic_params_t generic; /**< Parameters for defining generic header removal manipulation, + relevant if type = e_IOC_FM_PCD_MANIP_RMV_GENERIC */ + } u; +} ioc_fm_pcd_manip_hdr_rmv_params_t; + +/**************************************************************************//** + @Description Parameters for defining header manipulation node +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_hdr_params_t { + bool rmv; /**< TRUE, to define removal manipulation */ + ioc_fm_pcd_manip_hdr_rmv_params_t rmv_params; /**< Parameters for removal manipulation, relevant if 'rmv' = TRUE */ + + bool insrt; /**< TRUE, to define insertion manipulation */ + ioc_fm_pcd_manip_hdr_insrt_params_t insrt_params; /**< Parameters for insertion manipulation, relevant if 'insrt' = TRUE */ + + bool field_update; /**< TRUE, to define field update manipulation */ + ioc_fm_pcd_manip_hdr_field_update_params_t field_update_params; /**< Parameters for field update manipulation, relevant if 'fieldUpdate' = TRUE */ + + bool custom; /**< TRUE, to define custom manipulation */ + ioc_fm_pcd_manip_hdr_custom_params_t custom_params; /**< Parameters for custom manipulation, relevant if 'custom' = TRUE */ + + bool dont_parse_after_manip;/**< FALSE to activate the parser a second time after + completing the manipulation on the frame */ +} ioc_fm_pcd_manip_hdr_params_t; + + +/**************************************************************************//** + @Description structure for defining fragmentation manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_frag_params_t { + ioc_net_header_type hdr; /**< Header selection */ + union { + ioc_fm_pcd_manip_frag_capwap_params_t capwap_frag; /**< Parameters for defining CAPWAP fragmentation, + relevant if 'hdr' = HEADER_TYPE_CAPWAP */ + ioc_fm_pcd_manip_frag_ip_params_t ip_frag; /**< Parameters for defining IP fragmentation, + relevant if 'hdr' = HEADER_TYPE_Ipv4 or HEADER_TYPE_Ipv6 */ + } u; +} ioc_fm_pcd_manip_frag_params_t; + +/**************************************************************************//** + @Description structure for defining reassemble manipulation +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_reassem_params_t { + ioc_net_header_type hdr; /**< Header selection */ + union { + ioc_fm_pcd_manip_reassem_capwap_params_t capwap_reassem; /**< Parameters for defining CAPWAP reassembly, + relevant if 'hdr' = HEADER_TYPE_CAPWAP */ + ioc_fm_pcd_manip_reassem_ip_params_t ip_reassem; /**< Parameters for defining IP reassembly, + relevant if 'hdr' = HEADER_TYPE_Ipv4 or HEADER_TYPE_Ipv6 */ + } u; +} ioc_fm_pcd_manip_reassem_params_t; + +/**************************************************************************//** + @Description Parameters for defining a manipulation node +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_params_t { + ioc_fm_pcd_manip_type type; /**< Selects type of manipulation node */ + union { + ioc_fm_pcd_manip_hdr_params_t hdr; /**< Parameters for defining header manipulation node */ + ioc_fm_pcd_manip_reassem_params_t reassem;/**< Parameters for defining reassembly manipulation node */ + ioc_fm_pcd_manip_frag_params_t frag; /**< Parameters for defining fragmentation manipulation node */ + ioc_fm_pcd_manip_special_offload_params_t special_offload;/**< Parameters for defining special offload manipulation node */ + } u; + void *p_next_manip;/**< Handle to another (previously defined) manipulation node; + Allows concatenation of manipulation actions + This parameter is optional and may be NULL. */ + void *id; +} ioc_fm_pcd_manip_params_t; + +/**************************************************************************//** + @Description Structure for retrieving IP reassembly statistics +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_reassem_ip_stats_t { + /* common counters for both IPv4 and IPv6 */ + uint32_t timeout; /**< Counts the number of TimeOut occurrences */ + uint32_t rfd_pool_busy; /**< Counts the number of failed attempts to allocate + a Reassembly Frame Descriptor */ + uint32_t internal_buffer_busy; /**< Counts the number of times an internal buffer busy occurred */ + uint32_t external_buffer_busy; /**< Counts the number of times external buffer busy occurred */ + uint32_t sg_fragments; /**< Counts the number of Scatter/Gather fragments */ + uint32_t dma_semaphore_depletion; /**< Counts the number of failed attempts to allocate a DMA semaphore */ + uint32_t non_consistent_sp; /**< Counts the number of Non Consistent Storage Profile events for + successfully reassembled frames */ +struct { + uint32_t successfully_reassembled; /**< Counts the number of successfully reassembled frames */ + uint32_t valid_fragments; /**< Counts the total number of valid fragments that + have been processed for all frames */ + uint32_t processed_fragments; /**< Counts the number of processed fragments + (valid and error fragments) for all frames */ + uint32_t malformed_fragments; /**< Counts the number of malformed fragments processed for all frames */ + uint32_t discarded_fragments; /**< Counts the number of fragments discarded by the reassembly process */ + uint32_t auto_learn_busy; /**< Counts the number of times a busy condition occurs when attempting + to access an IP-Reassembly Automatic Learning Hash set */ + uint32_t more_than16fragments; /**< Counts the fragment occurrences in which the number of fragments-per-frame + exceeds 16 */ + } specific_hdr_statistics[2]; /**< slot '0' is for IPv4, slot '1' is for IPv6 */ +} ioc_fm_pcd_manip_reassem_ip_stats_t; + +/**************************************************************************//** + @Description Structure for retrieving IP fragmentation statistics +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_frag_ip_stats_t { + uint32_t total_frames; /**< Number of frames that passed through the manipulation node */ + uint32_t fragmented_frames; /**< Number of frames that were fragmented */ + uint32_t generated_fragments; /**< Number of fragments that were generated */ +} ioc_fm_pcd_manip_frag_ip_stats_t; + +/**************************************************************************//** + @Description Structure for retrieving CAPWAP reassembly statistics +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_reassem_capwap_stats_t { + uint32_t timeout; /**< Counts the number of timeout occurrences */ + uint32_t rfd_pool_busy; /**< Counts the number of failed attempts to allocate + a Reassembly Frame Descriptor */ + uint32_t internal_buffer_busy; /**< Counts the number of times an internal buffer busy occurred */ + uint32_t external_buffer_busy; /**< Counts the number of times external buffer busy occurred */ + uint32_t sg_fragments; /**< Counts the number of Scatter/Gather fragments */ + uint32_t dma_semaphore_depletion; /**< Counts the number of failed attempts to allocate a DMA semaphore */ + uint32_t successfully_reassembled; /**< Counts the number of successfully reassembled frames */ + uint32_t valid_fragments; /**< Counts the total number of valid fragments that + have been processed for all frames */ + uint32_t processed_fragments; /**< Counts the number of processed fragments + (valid and error fragments) for all frames */ + uint32_t malformed_fragments; /**< Counts the number of malformed fragments processed for all frames */ + uint32_t autoLearn_busy; /**< Counts the number of times a busy condition occurs when attempting + to access an Reassembly Automatic Learning Hash set */ + uint32_t discarded_fragments; /**< Counts the number of fragments discarded by the reassembly process */ + uint32_t more_than16fragments; /**< Counts the fragment occurrences in which the number of fragments-per-frame + exceeds 16 */ + uint32_t exceed_max_reassembly_frame_len;/**< ounts the number of times that a successful reassembled frame + length exceeds MaxReassembledFrameLength value */ +} ioc_fm_pcd_manip_reassem_capwap_stats_t; + +/**************************************************************************//** + @Description Structure for retrieving CAPWAP fragmentation statistics +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_frag_capwap_stats_t { + uint32_t total_frames; /**< Number of frames that passed through the manipulation node */ + uint32_t fragmented_frames; /**< Number of frames that were fragmented */ + uint32_t generated_fragments; /**< Number of fragments that were generated */ +#if (defined(DEBUG_ERRORS) && (DEBUG_ERRORS > 0)) + uint8_t sg_allocation_failure; /**< Number of allocation failure of s/g buffers */ +#endif /* (defined(DEBUG_ERRORS) && (DEBUG_ERRORS > 0)) */ +} ioc_fm_pcd_manip_frag_capwap_stats_t; + +/**************************************************************************//** + @Description Structure for retrieving reassembly statistics +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_reassem_stats_t { + union { + ioc_fm_pcd_manip_reassem_ip_stats_t ip_reassem; /**< Structure for IP reassembly statistics */ + ioc_fm_pcd_manip_reassem_capwap_stats_t capwap_reassem; /**< Structure for CAPWAP reassembly statistics */ + } u; +} ioc_fm_pcd_manip_reassem_stats_t; + +/**************************************************************************//** + @Description structure for retrieving fragmentation statistics +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_frag_stats_t { + union { + ioc_fm_pcd_manip_frag_ip_stats_t ip_frag; /**< Structure for IP fragmentation statistics */ + ioc_fm_pcd_manip_frag_capwap_stats_t capwap_frag; /**< Structure for CAPWAP fragmentation statistics */ + } u; +} ioc_fm_pcd_manip_frag_stats_t; + +/**************************************************************************//** + @Description structure for defining manipulation statistics +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_stats_t { + union { + ioc_fm_pcd_manip_reassem_stats_t reassem; /**< Structure for reassembly statistics */ + ioc_fm_pcd_manip_frag_stats_t frag; /**< Structure for fragmentation statistics */ + } u; +} ioc_fm_pcd_manip_stats_t; + +/**************************************************************************//** + @Description Parameters for acquiring manipulation statistics +*//***************************************************************************/ +typedef struct ioc_fm_pcd_manip_get_stats_t { + void *id; + ioc_fm_pcd_manip_stats_t stats; +} ioc_fm_pcd_manip_get_stats_t; + +/**************************************************************************//** + @Description Parameters for defining frame replicator group and its members +*//***************************************************************************/ +typedef struct ioc_fm_pcd_frm_replic_group_params_t { + uint8_t max_num_of_entries; /**< Maximal number of members in the group - must be at least two */ + uint8_t num_of_entries; /**< Number of members in the group - must be at least 1 */ + ioc_fm_pcd_cc_next_engine_params_t next_engine_params[IOC_FM_PCD_FRM_REPLIC_MAX_NUM_OF_ENTRIES]; + /**< Array of members' parameters */ + void *id; +} ioc_fm_pcd_frm_replic_group_params_t; + +typedef struct ioc_fm_pcd_frm_replic_member_t { + void *h_replic_group; + uint16_t member_index; +} ioc_fm_pcd_frm_replic_member_t; + +typedef struct ioc_fm_pcd_frm_replic_member_params_t { + ioc_fm_pcd_frm_replic_member_t member; + ioc_fm_pcd_cc_next_engine_params_t next_engine_params; +} ioc_fm_pcd_frm_replic_member_params_t; + + +typedef struct ioc_fm_pcd_cc_key_statistics_t { + uint32_t byte_count; /**< This counter reflects byte count of frames that + were matched by this key. */ + uint32_t frame_count; /**< This counter reflects count of frames that + were matched by this key. */ + uint32_t frame_length_range_count[IOC_FM_PCD_CC_STATS_MAX_NUM_OF_FLR]; + /**< These counters reflect how many frames matched + this key in 'RMON' statistics mode: + Each counter holds the number of frames of a + specific frames length range, according to the + ranges provided at initialization. */ +} ioc_fm_pcd_cc_key_statistics_t; + + +typedef struct ioc_fm_pcd_cc_tbl_get_stats_t { + void *id; + uint16_t key_index; + ioc_fm_pcd_cc_key_statistics_t statistics; +} ioc_fm_pcd_cc_tbl_get_stats_t; + +/**************************************************************************//** + @Function FM_PCD_MatchTableGetKeyStatistics + + @Description This routine may be used to get statistics counters of specific key + in a CC Node. + + If 'e_FM_PCD_CC_STATS_MODE_FRAME' and + 'e_FM_PCD_CC_STATS_MODE_BYTE_AND_FRAME' were set for this node, + these counters reflect how many frames passed that were matched + this key; The total frames count will be returned in the counter + of the first range (as only one frame length range was defined). + If 'e_FM_PCD_CC_STATS_MODE_RMON' was set for this node, the total + frame count will be separated to frame length counters, based on + provided frame length ranges. + + @Param[in] h_CcNode A handle to the node + @Param[in] keyIndex Key index for adding + @Param[out] p_KeyStatistics Key statistics counters + + @Return The specific key statistics. + + @Cautions Allowed only following FM_PCD_MatchTableSet(). +*//***************************************************************************/ + +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_GET_KEY_STAT_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(12), ioc_compat_fm_pcd_cc_tbl_get_stats_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_GET_KEY_STAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(12), ioc_fm_pcd_cc_tbl_get_stats_t) + +/**************************************************************************//** + @Function FM_PCD_MatchTableGetMissStatistics + + @Description This routine may be used to get statistics counters of miss entry + in a CC Node. + + If 'e_FM_PCD_CC_STATS_MODE_FRAME' and + 'e_FM_PCD_CC_STATS_MODE_BYTE_AND_FRAME' were set for this node, + these counters reflect how many frames were not matched to any + existing key and therefore passed through the miss entry; The + total frames count will be returned in the counter of the + first range (as only one frame length range was defined). + + @Param[in] h_CcNode A handle to the node + @Param[out] p_MissStatistics Statistics counters for 'miss' + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_MatchTableSet(). +*//***************************************************************************/ + +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_GET_MISS_STAT_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(13), ioc_compat_fm_pcd_cc_tbl_get_stats_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_GET_MISS_STAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(13), ioc_fm_pcd_cc_tbl_get_stats_t) + +/**************************************************************************//** + @Function FM_PCD_HashTableGetMissStatistics + + @Description This routine may be used to get statistics counters of 'miss' + entry of the a hash table. + + If 'e_FM_PCD_CC_STATS_MODE_FRAME' and + 'e_FM_PCD_CC_STATS_MODE_BYTE_AND_FRAME' were set for this node, + these counters reflect how many frames were not matched to any + existing key and therefore passed through the miss entry; + + @Param[in] h_HashTbl A handle to a hash table + @Param[out] p_MissStatistics Statistics counters for 'miss' + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_HashTableSet(). +*//***************************************************************************/ + +#if defined(FM_COMPAT) +#define FM_PCD_IOC_HASH_TABLE_GET_MISS_STAT_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(14), ioc_compat_fm_pcd_cc_tbl_get_stats_t) +#endif +#define FM_PCD_IOC_HASH_TABLE_GET_MISS_STAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(14), ioc_fm_pcd_cc_tbl_get_stats_t) + + +/**************************************************************************//** + @Function FM_PCD_NetEnvCharacteristicsSet + + @Description Define a set of Network Environment Characteristics. + + When setting an environment it is important to understand its + application. It is not meant to describe the flows that will run + on the ports using this environment, but what the user means TO DO + with the PCD mechanisms in order to parse-classify-distribute those + frames. + By specifying a distinction unit, the user means it would use that option + for distinction between frames at either a KeyGen scheme or a coarse + classification action descriptor. Using interchangeable headers to define a + unit means that the user is indifferent to which of the interchangeable + headers is present in the frame, and wants the distinction to be based + on the presence of either one of them. + + Depending on context, there are limitations to the use of environments. A + port using the PCD functionality is bound to an environment. Some or even + all ports may share an environment but also an environment per port is + possible. When initializing a scheme, a classification plan group (see below), + or a coarse classification tree, one of the initialized environments must be + stated and related to. When a port is bound to a scheme, a classification + plan group, or a coarse classification tree, it MUST be bound to the same + environment. + + The different PCD modules, may relate (for flows definition) ONLY on + distinction units as defined by their environment. When initializing a + scheme for example, it may not choose to select IPV4 as a match for + recognizing flows unless it was defined in the relating environment. In + fact, to guide the user through the configuration of the PCD, each module's + characterization in terms of flows is not done using protocol names, but using + environment indexes. + + In terms of HW implementation, the list of distinction units sets the LCV vectors + and later used for match vector, classification plan vectors and coarse classification + indexing. + + @Param[in,out] ioc_fm_pcd_net_env_params_t A structure defining the distiction units for this configuration. + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_NET_ENV_CHARACTERISTICS_SET_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(20), ioc_compat_fm_pcd_net_env_params_t) +#endif +#define FM_PCD_IOC_NET_ENV_CHARACTERISTICS_SET _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(20), ioc_fm_pcd_net_env_params_t) + +/**************************************************************************//** + @Function FM_PCD_NetEnvCharacteristicsDelete + + @Description Deletes a set of Network Environment Charecteristics. + + @Param[in] ioc_fm_obj_t - The id of a Network Environment object. + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_NET_ENV_CHARACTERISTICS_DELETE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(21), ioc_compat_fm_obj_t) +#endif +#define FM_PCD_IOC_NET_ENV_CHARACTERISTICS_DELETE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(21), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_PCD_KgSchemeSet + + @Description Initializing or modifying and enabling a scheme for the KeyGen. + This routine should be called for adding or modifying a scheme. + When a scheme needs modifying, the API requires that it will be + rewritten. In such a case 'modify' should be TRUE. If the + routine is called for a valid scheme and 'modify' is FALSE, + it will return error. + + @Param[in,out] ioc_fm_pcd_kg_scheme_params_t A structure of parameters for defining the scheme + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_KG_SCHEME_SET_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(24), ioc_compat_fm_pcd_kg_scheme_params_t) +#endif +#define FM_PCD_IOC_KG_SCHEME_SET _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(24), ioc_fm_pcd_kg_scheme_params_t) + +/**************************************************************************//** + @Function FM_PCD_KgSchemeDelete + + @Description Deleting an initialized scheme. + + @Param[in] ioc_fm_obj_t scheme id as initalized by application at FM_PCD_IOC_KG_SET_SCHEME + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_KG_SCHEME_DELETE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(25), ioc_compat_fm_obj_t) +#endif +#define FM_PCD_IOC_KG_SCHEME_DELETE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(25), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_PCD_CcRootBuild + + @Description This routine must be called to define a complete coarse + classification tree. This is the way to define coarse + classification to a certain flow - the KeyGen schemes + may point only to trees defined in this way. + + @Param[in,out] ioc_fm_pcd_cc_tree_params_t A structure of parameters to define the tree. + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_CC_ROOT_BUILD_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(26), compat_uptr_t) +#endif +#if defined(__FreeBSD__) +/* + * FreeBSD ioctl framework copies IOCPARM_LEN(cmd) bytes in/out automatically. + * ioc_fm_pcd_cc_tree_params_t exceeds IOCPARM_MAX (8192), so use _IO to pass + * the userspace address directly. The kernel handler does copyin/copyout. + */ +#define FM_PCD_IOC_CC_ROOT_BUILD _IO(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(26)) +#else +#define FM_PCD_IOC_CC_ROOT_BUILD _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(26), void *) /* workaround ...*/ +#endif + +/**************************************************************************//** + @Function FM_PCD_CcRootDelete + + @Description Deleting a built tree. + + @Param[in] ioc_fm_obj_t - The id of a CC tree. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_CC_ROOT_DELETE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(27), ioc_compat_fm_obj_t) +#endif +#define FM_PCD_IOC_CC_ROOT_DELETE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(27), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_PCD_MatchTableSet + + @Description This routine should be called for each CC (coarse classification) + node. The whole CC tree should be built bottom up so that each + node points to already defined nodes. p_NodeId returns the node + Id to be used by other nodes. + + @Param[in,out] ioc_fm_pcd_cc_node_params_t A structure for defining the CC node params + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_SET_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(28), compat_uptr_t) +#endif +#if defined(__FreeBSD__) +/* See FM_PCD_IOC_CC_ROOT_BUILD comment — same large-struct issue. */ +#define FM_PCD_IOC_MATCH_TABLE_SET _IO(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(28)) +#else +#define FM_PCD_IOC_MATCH_TABLE_SET _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(28), void *) /* workaround ...*/ +#endif + +/**************************************************************************//** + @Function FM_PCD_MatchTableDelete + + @Description Deleting a built node. + + @Param[in] ioc_fm_obj_t - The id of a CC node. + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_DELETE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(29), ioc_compat_fm_obj_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_DELETE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(29), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_PCD_CcRootModifyNextEngine + + @Description Modify the Next Engine Parameters in the entry of the tree. + + @Param[in] ioc_fm_pcd_cc_tree_modify_next_engine_params_t - Pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_CcRootBuild(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_CC_ROOT_MODIFY_NEXT_ENGINE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(30), ioc_compat_fm_pcd_cc_tree_modify_next_engine_params_t) +#endif +#define FM_PCD_IOC_CC_ROOT_MODIFY_NEXT_ENGINE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(30), ioc_fm_pcd_cc_tree_modify_next_engine_params_t) + +/**************************************************************************//** + @Function FM_PCD_MatchTableModifyNextEngine + + @Description Modify the Next Engine Parameters in the relevant key entry of the node. + + @Param[in] ioc_fm_pcd_cc_node_modify_next_engine_params_t A pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_MatchTableSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_MODIFY_NEXT_ENGINE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(31), ioc_compat_fm_pcd_cc_node_modify_next_engine_params_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_MODIFY_NEXT_ENGINE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(31), ioc_fm_pcd_cc_node_modify_next_engine_params_t) + +/**************************************************************************//** + @Function FM_PCD_MatchTableModifyMissNextEngine + + @Description Modify the Next Engine Parameters of the Miss key case of the node. + + @Param[in] ioc_fm_pcd_cc_node_modify_next_engine_params_t - Pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_MatchTableSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_MODIFY_MISS_NEXT_ENGINE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(32), ioc_compat_fm_pcd_cc_node_modify_next_engine_params_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_MODIFY_MISS_NEXT_ENGINE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(32), ioc_fm_pcd_cc_node_modify_next_engine_params_t) + +/**************************************************************************//** + @Function FM_PCD_MatchTableRemoveKey + + @Description Remove the key (including next engine parameters of this key) + defined by the index of the relevant node. + + @Param[in] ioc_fm_pcd_cc_node_remove_key_params_t A pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only after FM_PCD_MatchTableSet() has been called for this + node and for all of the nodes that lead to it. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_REMOVE_KEY_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(33), ioc_compat_fm_pcd_cc_node_remove_key_params_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_REMOVE_KEY _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(33), ioc_fm_pcd_cc_node_remove_key_params_t) + +/**************************************************************************//** + @Function FM_PCD_MatchTableAddKey + + @Description Add the key (including next engine parameters of this key in the + index defined by the keyIndex. Note that 'FM_PCD_LAST_KEY_INDEX' + may be used when the user doesn't care about the position of the + key in the table - in that case, the key will be automatically + added by the driver in the last available entry. + + @Param[in] ioc_fm_pcd_cc_node_modify_key_and_next_engine_params_t A pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only after FM_PCD_MatchTableSet() has been called for this + node and for all of the nodes that lead to it. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_ADD_KEY_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(34), ioc_compat_fm_pcd_cc_node_modify_key_and_next_engine_params_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_ADD_KEY _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(34), ioc_fm_pcd_cc_node_modify_key_and_next_engine_params_t) + +/**************************************************************************//** + @Function FM_PCD_MatchTableModifyKeyAndNextEngine + + @Description Modify the key and Next Engine Parameters of this key in the index defined by key_index. + + @Param[in] ioc_fm_pcd_cc_node_modify_key_and_next_engine_params_t A pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_MatchTableSet() not only of the relevnt node but also + the node that points to this node +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_MODIFY_KEY_AND_NEXT_ENGINE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(35), ioc_compat_fm_pcd_cc_node_modify_key_and_next_engine_params_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_MODIFY_KEY_AND_NEXT_ENGINE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(35), ioc_fm_pcd_cc_node_modify_key_and_next_engine_params_t) + +/**************************************************************************//** + @Function FM_PCD_MatchTableModifyKey + + @Description Modify the key at the index defined by key_index. + + @Param[in] ioc_fm_pcd_cc_node_modify_key_params_t - Pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only after FM_PCD_MatchTableSet() has been called for this + node and for all of the nodes that lead to it. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MATCH_TABLE_MODIFY_KEY_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(36), ioc_compat_fm_pcd_cc_node_modify_key_params_t) +#endif +#define FM_PCD_IOC_MATCH_TABLE_MODIFY_KEY _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(36), ioc_fm_pcd_cc_node_modify_key_params_t) + +/**************************************************************************//** + @Function FM_PCD_HashTableSet + + @Description This routine initializes a hash table structure. + KeyGen hash result determines the hash bucket. + Next, KeyGen key is compared against all keys of this + bucket (exact match). + Number of sets (number of buckets) of the hash equals to the + number of 1-s in 'hash_res_mask' in the provided parameters. + Number of hash table ways is then calculated by dividing + 'max_num_of_keys' equally between the hash sets. This is the maximal + number of keys that a hash bucket may hold. + The hash table is initialized empty and keys may be + added to it following the initialization. Keys masks are not + supported in current hash table implementation. + The initialized hash table can be integrated as a node in a + CC tree. + + @Param[in,out] ioc_fm_pcd_hash_table_params_t - Pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_HASH_TABLE_SET_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(37), ioc_compat_fm_pcd_hash_table_params_t) +#endif +#define FM_PCD_IOC_HASH_TABLE_SET _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(37), ioc_fm_pcd_hash_table_params_t) + + +/**************************************************************************//** + @Function FM_PCD_HashTableDelete + + @Description This routine deletes the provided hash table and released all + its allocated resources. + + @Param[in] ioc_fm_obj_t - The ID of a hash table. + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_HashTableSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_HASH_TABLE_DELETE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(37), ioc_compat_fm_obj_t) +#endif +#define FM_PCD_IOC_HASH_TABLE_DELETE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(37), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_PCD_HashTableAddKey + + @Description This routine adds the provided key (including next engine + parameters of this key) to the hash table. + The key is added as the last key of the bucket that it is + mapped to. + + @Param[in] ioc_fm_pcd_hash_table_add_key_params_t - Pointer to a structure with the relevant parameters + + @Return 0 on success; error code otherwise. + + @Cautions Allowed only following FM_PCD_HashTableSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_HASH_TABLE_ADD_KEY_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(39), ioc_compat_fm_pcd_hash_table_add_key_params_t) +#endif +#define FM_PCD_IOC_HASH_TABLE_ADD_KEY _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(39), ioc_fm_pcd_hash_table_add_key_params_t) + +/**************************************************************************//** + @Function FM_PCD_HashTableRemoveKey + + @Description This routine removes the requested key (including next engine + parameters of this key) from the hash table. + + @Param[in] ioc_fm_pcd_hash_table_remove_key_params_t - Pointer to a structure with the relevant parameters + + @Return 0 on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_HashTableSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_HASH_TABLE_REMOVE_KEY_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(40), ioc_compat_fm_pcd_hash_table_remove_key_params_t) +#endif +#define FM_PCD_IOC_HASH_TABLE_REMOVE_KEY _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(40), ioc_fm_pcd_hash_table_remove_key_params_t) + +/**************************************************************************//** + @Function FM_PCD_PlcrProfileSet + + @Description Sets a profile entry in the policer profile table. + The routine overrides any existing value. + + @Param[in,out] ioc_fm_pcd_plcr_profile_params_t A structure of parameters for defining a + policer profile entry. + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_PLCR_PROFILE_SET_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(41), ioc_compat_fm_pcd_plcr_profile_params_t) +#endif +#define FM_PCD_IOC_PLCR_PROFILE_SET _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(41), ioc_fm_pcd_plcr_profile_params_t) + +/**************************************************************************//** + @Function FM_PCD_PlcrProfileDelete + + @Description Delete a profile entry in the policer profile table. + The routine set entry to invalid. + + @Param[in] ioc_fm_obj_t The id of a policer profile. + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_PLCR_PROFILE_DELETE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(41), ioc_compat_fm_obj_t) +#endif +#define FM_PCD_IOC_PLCR_PROFILE_DELETE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(41), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_PCD_ManipNodeSet + + @Description This routine should be called for defining a manipulation + node. A manipulation node must be defined before the CC node + that precedes it. + + @Param[in] ioc_fm_pcd_manip_params_t - A structure of parameters defining the manipulation + + @Return A handle to the initialized object on success; NULL code otherwise. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MANIP_NODE_SET_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(43), ioc_compat_fm_pcd_manip_params_t) +#endif +#define FM_PCD_IOC_MANIP_NODE_SET _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(43), ioc_fm_pcd_manip_params_t) + +/**************************************************************************//** + @Function FM_PCD_ManipNodeReplace + + @Description Change existing manipulation node to be according to new requirement. + (Here, it's implemented as a variant of the same IOCTL as for + FM_PCD_ManipNodeSet(), and one that when called, the 'id' member + in its 'ioc_fm_pcd_manip_params_t' argument is set to contain + the manip node's handle) + + @Param[in] ioc_fm_pcd_manip_params_t - A structure of parameters defining the manipulation + + @Return 0 on success; error code otherwise. + + @Cautions Allowed only following FM_PCD_ManipNodeSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MANIP_NODE_REPLACE_COMPAT FM_PCD_IOC_MANIP_NODE_SET_COMPAT +#endif +#define FM_PCD_IOC_MANIP_NODE_REPLACE FM_PCD_IOC_MANIP_NODE_SET + +/**************************************************************************//** + @Function FM_PCD_ManipNodeDelete + + @Description Delete an existing manipulation node. + + @Param[in] ioc_fm_obj_t The id of the manipulation node to delete. + + @Return 0 on success; error code otherwise. + + @Cautions Allowed only following FM_PCD_ManipNodeSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MANIP_NODE_DELETE_COMPAT _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(44), ioc_compat_fm_obj_t) +#endif +#define FM_PCD_IOC_MANIP_NODE_DELETE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(44), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_PCD_ManipGetStatistics + + @Description Retrieve the manipulation statistics. + + @Param[in] h_ManipNode A handle to a manipulation node. + @Param[out] p_FmPcdManipStats A structure for retrieving the manipulation statistics + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_ManipNodeSet(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_MANIP_GET_STATS_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(50), ioc_compat_fm_pcd_manip_get_stats_t) +#endif +#define FM_PCD_IOC_MANIP_GET_STATS _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(50), ioc_fm_pcd_manip_get_stats_t) + +/**************************************************************************//** +@Function FM_PCD_SetAdvancedOffloadSupport + +@Description This routine must be called in order to support the following features: + IP-fragmentation, IP-reassembly, IPsec, Header-manipulation, frame-replicator. + +@Param[in] h_FmPcd FM PCD module descriptor. + +@Return 0 on success; error code otherwise. + +@Cautions Allowed only when PCD is disabled. +*//***************************************************************************/ +#define FM_PCD_IOC_SET_ADVANCED_OFFLOAD_SUPPORT _IO(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(45)) + +/**************************************************************************//** + @Function FM_PCD_AllowHcUsage + + @Description This routine must be called to temporary allow/deny HC usage + This is the way to avoid crash of certain HC operations + performed under heavy traffic. + + @Param[in] uint8_t 0 - Deny HC usage HC, 1 - Allow HC usage + + @Return 0 on success; Error code otherwise. +*//***************************************************************************/ +#define FM_PCD_IOC_ALLOW_HC_USAGE _IOW(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(51), uint8_t) + +/**************************************************************************//** + @Function FM_PCD_FrmReplicSetGroup + + @Description Initialize a Frame Replicator group. + + @Param[in] h_FmPcd FM PCD module descriptor. + @Param[in] p_FrmReplicGroupParam A structure of parameters for the initialization of + the frame replicator group. + + @Return A handle to the initialized object on success; NULL code otherwise. + + @Cautions Allowed only following FM_PCD_Init(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_FRM_REPLIC_GROUP_SET_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(46), ioc_compat_fm_pcd_frm_replic_group_params_t) +#endif +#if defined(__FreeBSD__) +/* Struct exceeds FreeBSD IOCPARM_MAX (4096); use _IO with manual copyin. */ +#define FM_PCD_IOC_FRM_REPLIC_GROUP_SET _IO(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(46)) +#else +#define FM_PCD_IOC_FRM_REPLIC_GROUP_SET _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(46), ioc_fm_pcd_frm_replic_group_params_t) +#endif + +/**************************************************************************//** + @Function FM_PCD_FrmReplicDeleteGroup + + @Description Delete a Frame Replicator group. + + @Param[in] h_FrmReplicGroup A handle to the frame replicator group. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_FrmReplicSetGroup(). +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_FRM_REPLIC_GROUP_DELETE_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(47), ioc_compat_fm_obj_t) +#endif +#define FM_PCD_IOC_FRM_REPLIC_GROUP_DELETE _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(47), ioc_fm_obj_t) + +/**************************************************************************//** + @Function FM_PCD_FrmReplicAddMember + + @Description Add the member in the index defined by the memberIndex. + + @Param[in] h_FrmReplicGroup A handle to the frame replicator group. + @Param[in] memberIndex member index for adding. + @Param[in] p_MemberParams A pointer to the new member parameters. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_FrmReplicSetGroup() of this group. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_FRM_REPLIC_MEMBER_ADD_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(48), ioc_compat_fm_pcd_frm_replic_member_params_t) +#endif +#define FM_PCD_IOC_FRM_REPLIC_MEMBER_ADD _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(48), ioc_fm_pcd_frm_replic_member_params_t) + +/**************************************************************************//** + @Function FM_PCD_FrmReplicRemoveMember + + @Description Remove the member defined by the index from the relevant group. + + @Param[in] h_FrmReplicGroup A handle to the frame replicator group. + @Param[in] memberIndex member index for removing. + + @Return E_OK on success; Error code otherwise. + + @Cautions Allowed only following FM_PCD_FrmReplicSetGroup() of this group. +*//***************************************************************************/ +#if defined(FM_COMPAT) +#define FM_PCD_IOC_FRM_REPLIC_MEMBER_REMOVE_COMPAT _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(49), ioc_compat_fm_pcd_frm_replic_member_t) +#endif +#define FM_PCD_IOC_FRM_REPLIC_MEMBER_REMOVE _IOWR(FM_IOC_TYPE_BASE, FM_PCD_IOC_NUM(49), ioc_fm_pcd_frm_replic_member_t) + +#endif /* __FM_PCD_IOCTLS_H */ +/** @} */ /* end of lnx_ioctl_FM_PCD_Runtime_grp group */ +/** @} */ /* end of lnx_ioctl_FM_PCD_grp group */ +/** @} */ /* end of lnx_ioctl_FM_grp group */ diff --git a/sys/contrib/ncsw/inc/ioctls/integration_ioctls.h b/sys/contrib/ncsw/inc/ioctls/integration_ioctls.h new file mode 100644 index 000000000000..61d696e2de99 --- /dev/null +++ b/sys/contrib/ncsw/inc/ioctls/integration_ioctls.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2008-2012 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/**************************************************************************//** + @File integration_ioctls.h + + @Description External header file for Integration unit routines. +*//***************************************************************************/ + +#ifndef __INTG_IOCTLS_H +#define __INTG_IOCTLS_H + + +#define FM_IOC_TYPE_BASE (NCSW_IOC_TYPE_BASE+1) +#define FMT_IOC_TYPE_BASE (NCSW_IOC_TYPE_BASE+3) + +/*#define FM_IOCTL_DBG*/ + +#if defined(FM_IOCTL_DBG) + #define _fm_ioctl_dbg(format, arg...) \ + printk("fm ioctl [%s:%u](cpu:%u) - " format, \ + __func__, __LINE__, smp_processor_id(), ##arg) +#else +# define _fm_ioctl_dbg(arg...) +#endif + +#endif /* __INTG_IOCTLS_H */ diff --git a/sys/contrib/ncsw/inc/ioctls/ioctls.h b/sys/contrib/ncsw/inc/ioctls/ioctls.h new file mode 100644 index 000000000000..495d2e6f6624 --- /dev/null +++ b/sys/contrib/ncsw/inc/ioctls/ioctls.h @@ -0,0 +1,96 @@ +/* Copyright (c) 2008-2012 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/**************************************************************************//** + @File ioctls.h + + @Description Structures and definitions for Command Relay Ioctls +*//***************************************************************************/ + +#ifndef __IOCTLS_H__ +#define __IOCTLS_H__ + +#include + +#include "integration_ioctls.h" + + +/**************************************************************************//** + @Group lnx_ioctl_ncsw_grp NetCommSw Linux User-Space (IOCTL) API + @{ +*//***************************************************************************/ + +#define NCSW_IOC_TYPE_BASE 0xe0 /**< defines the IOCTL type for all + the NCSW Linux module commands */ + + +/**************************************************************************//** + @Description IOCTL Memory allocation types. +*//***************************************************************************/ +typedef enum ioc_mem_type { + e_IOC_MEM_INVALID = 0x00000000, /**< Invalid memory type (error) */ + e_IOC_MEM_CACHABLE_SYS = 0x00000001, /**< Primary DDR, cacheable segment */ + e_IOC_MEM_NOCACHE_SYS = 0x00000004, /**< Primary DDR, non-cacheable segment */ + e_IOC_MEM_SECONDARY = 0x00000002, /**< Either secondary DDR or SDRAM */ + e_IOC_MEM_PRAM = 0x00000008 /**< Multi-user RAM identifier */ +} ioc_mem_type; + +/**************************************************************************//** + @Description Enumeration (bit flags) of communication modes (Transmit, + receive or both). +*//***************************************************************************/ +typedef enum ioc_comm_mode { + e_IOC_COMM_MODE_NONE = 0 /**< No transmit/receive communication */ + , e_IOC_COMM_MODE_RX = 1 /**< Only receive communication */ + , e_IOC_COMM_MODE_TX = 2 /**< Only transmit communication */ + , e_IOC_COMM_MODE_RX_AND_TX = 3 /**< Both transmit and receive communication */ +} ioc_comm_mode; + +/**************************************************************************//** + @Description General Diagnostic Mode +*//***************************************************************************/ +typedef enum ioc_diag_mode +{ + e_IOC_DIAG_MODE_NONE = 0, + e_IOC_DIAG_MODE_CTRL_LOOPBACK, /**< loopback in the controller; E.g. MAC, TDM, etc. */ + e_IOC_DIAG_MODE_CHIP_LOOPBACK, /**< loopback in the chip but not in controller; + E.g. IO-pins, SerDes, etc. */ + e_IOC_DIAG_MODE_PHY_LOOPBACK, /**< loopback in the external PHY */ + e_IOC_DIAG_MODE_LINE_LOOPBACK, /**< loopback in the external line */ + e_IOC_DIAG_MODE_CTRL_ECHO, /**< */ + e_IOC_DIAG_MODE_PHY_ECHO /**< */ +} ioc_diag_mode; + +/** @} */ /* end of lnx_ioctl_ncsw_grp */ + + +#endif /* __IOCTLS_H__ */ diff --git a/sys/contrib/ncsw/inc/ioctls/net_ioctls.h b/sys/contrib/ncsw/inc/ioctls/net_ioctls.h new file mode 100644 index 000000000000..c99d64cfc28f --- /dev/null +++ b/sys/contrib/ncsw/inc/ioctls/net_ioctls.h @@ -0,0 +1,430 @@ +/* Copyright (c) 2008-2012 Freescale Semiconductor, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +/**************************************************************************//** + @File net_ioctls.h + + @Description This file contains common and general netcomm headers definitions. +*//***************************************************************************/ +#ifndef __NET_IOCTLS_H +#define __NET_IOCTLS_H + +#include "ioctls.h" + + +typedef uint8_t ioc_header_field_ppp_t; + +#define IOC_NET_HEADER_FIELD_PPP_PID (1) +#define IOC_NET_HEADER_FIELD_PPP_COMPRESSED (IOC_NET_HEADER_FIELD_PPP_PID << 1) +#define IOC_NET_HEADER_FIELD_PPP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_PPP_PID << 2) - 1) + + +typedef uint8_t ioc_header_field_pppoe_t; + +#define IOC_NET_HEADER_FIELD_PPPoE_VER (1) +#define IOC_NET_HEADER_FIELD_PPPoE_TYPE (IOC_NET_HEADER_FIELD_PPPoE_VER << 1) +#define IOC_NET_HEADER_FIELD_PPPoE_CODE (IOC_NET_HEADER_FIELD_PPPoE_VER << 2) +#define IOC_NET_HEADER_FIELD_PPPoE_SID (IOC_NET_HEADER_FIELD_PPPoE_VER << 3) +#define IOC_NET_HEADER_FIELD_PPPoE_LEN (IOC_NET_HEADER_FIELD_PPPoE_VER << 4) +#define IOC_NET_HEADER_FIELD_PPPoE_SESSION (IOC_NET_HEADER_FIELD_PPPoE_VER << 5) +#define IOC_NET_HEADER_FIELD_PPPoE_PID (IOC_NET_HEADER_FIELD_PPPoE_VER << 6) +#define IOC_NET_HEADER_FIELD_PPPoE_ALL_FIELDS ((IOC_NET_HEADER_FIELD_PPPoE_VER << 7) - 1) + +#define IOC_NET_HEADER_FIELD_PPPMUX_PID (1) +#define IOC_NET_HEADER_FIELD_PPPMUX_CKSUM (IOC_NET_HEADER_FIELD_PPPMUX_PID << 1) +#define IOC_NET_HEADER_FIELD_PPPMUX_COMPRESSED (IOC_NET_HEADER_FIELD_PPPMUX_PID << 2) +#define IOC_NET_HEADER_FIELD_PPPMUX_ALL_FIELDS ((IOC_NET_HEADER_FIELD_PPPMUX_PID << 3) - 1) + +#define IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_PFF (1) +#define IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_LXT (IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_PFF << 1) +#define IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_LEN (IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_PFF << 2) +#define IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_PID (IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_PFF << 3) +#define IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_USE_PID (IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_PFF << 4) +#define IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_ALL_FIELDS ((IOC_NET_HEADER_FIELD_PPPMUX_SUBFRAME_PFF << 5) - 1) + + +typedef uint8_t ioc_header_field_eth_t; + +#define IOC_NET_HEADER_FIELD_ETH_DA (1) +#define IOC_NET_HEADER_FIELD_ETH_SA (IOC_NET_HEADER_FIELD_ETH_DA << 1) +#define IOC_NET_HEADER_FIELD_ETH_LENGTH (IOC_NET_HEADER_FIELD_ETH_DA << 2) +#define IOC_NET_HEADER_FIELD_ETH_TYPE (IOC_NET_HEADER_FIELD_ETH_DA << 3) +#define IOC_NET_HEADER_FIELD_ETH_FINAL_CKSUM (IOC_NET_HEADER_FIELD_ETH_DA << 4) +#define IOC_NET_HEADER_FIELD_ETH_PADDING (IOC_NET_HEADER_FIELD_ETH_DA << 5) +#define IOC_NET_HEADER_FIELD_ETH_ALL_FIELDS ((IOC_NET_HEADER_FIELD_ETH_DA << 6) - 1) + +#define IOC_NET_HEADER_FIELD_ETH_ADDR_SIZE 6 + +typedef uint16_t ioc_header_field_ip_t; + +#define IOC_NET_HEADER_FIELD_IP_VER (1) +#define IOC_NET_HEADER_FIELD_IP_DSCP (IOC_NET_HEADER_FIELD_IP_VER << 2) +#define IOC_NET_HEADER_FIELD_IP_ECN (IOC_NET_HEADER_FIELD_IP_VER << 3) +#define IOC_NET_HEADER_FIELD_IP_PROTO (IOC_NET_HEADER_FIELD_IP_VER << 4) + +#define IOC_NET_HEADER_FIELD_IP_PROTO_SIZE 1 + +typedef uint16_t ioc_header_field_ipv4_t; + +#define IOC_NET_HEADER_FIELD_IPv4_VER (1) +#define IOC_NET_HEADER_FIELD_IPv4_HDR_LEN (IOC_NET_HEADER_FIELD_IPv4_VER << 1) +#define IOC_NET_HEADER_FIELD_IPv4_TOS (IOC_NET_HEADER_FIELD_IPv4_VER << 2) +#define IOC_NET_HEADER_FIELD_IPv4_TOTAL_LEN (IOC_NET_HEADER_FIELD_IPv4_VER << 3) +#define IOC_NET_HEADER_FIELD_IPv4_ID (IOC_NET_HEADER_FIELD_IPv4_VER << 4) +#define IOC_NET_HEADER_FIELD_IPv4_FLAG_D (IOC_NET_HEADER_FIELD_IPv4_VER << 5) +#define IOC_NET_HEADER_FIELD_IPv4_FLAG_M (IOC_NET_HEADER_FIELD_IPv4_VER << 6) +#define IOC_NET_HEADER_FIELD_IPv4_OFFSET (IOC_NET_HEADER_FIELD_IPv4_VER << 7) +#define IOC_NET_HEADER_FIELD_IPv4_TTL (IOC_NET_HEADER_FIELD_IPv4_VER << 8) +#define IOC_NET_HEADER_FIELD_IPv4_PROTO (IOC_NET_HEADER_FIELD_IPv4_VER << 9) +#define IOC_NET_HEADER_FIELD_IPv4_CKSUM (IOC_NET_HEADER_FIELD_IPv4_VER << 10) +#define IOC_NET_HEADER_FIELD_IPv4_SRC_IP (IOC_NET_HEADER_FIELD_IPv4_VER << 11) +#define IOC_NET_HEADER_FIELD_IPv4_DST_IP (IOC_NET_HEADER_FIELD_IPv4_VER << 12) +#define IOC_NET_HEADER_FIELD_IPv4_OPTS (IOC_NET_HEADER_FIELD_IPv4_VER << 13) +#define IOC_NET_HEADER_FIELD_IPv4_OPTS_COUNT (IOC_NET_HEADER_FIELD_IPv4_VER << 14) +#define IOC_NET_HEADER_FIELD_IPv4_ALL_FIELDS ((IOC_NET_HEADER_FIELD_IPv4_VER << 15) - 1) + +#define IOC_NET_HEADER_FIELD_IPv4_ADDR_SIZE 4 +#define IOC_NET_HEADER_FIELD_IPv4_PROTO_SIZE 1 + + +typedef uint8_t ioc_header_field_ipv6_t; + +#define IOC_NET_HEADER_FIELD_IPv6_VER (1) +#define IOC_NET_HEADER_FIELD_IPv6_TC (IOC_NET_HEADER_FIELD_IPv6_VER << 1) +#define IOC_NET_HEADER_FIELD_IPv6_SRC_IP (IOC_NET_HEADER_FIELD_IPv6_VER << 2) +#define IOC_NET_HEADER_FIELD_IPv6_DST_IP (IOC_NET_HEADER_FIELD_IPv6_VER << 3) +#define IOC_NET_HEADER_FIELD_IPv6_NEXT_HDR (IOC_NET_HEADER_FIELD_IPv6_VER << 4) +#define IOC_NET_HEADER_FIELD_IPv6_FL (IOC_NET_HEADER_FIELD_IPv6_VER << 5) +#define IOC_NET_HEADER_FIELD_IPv6_HOP_LIMIT (IOC_NET_HEADER_FIELD_IPv6_VER << 6) +#define IOC_NET_HEADER_FIELD_IPv6_ALL_FIELDS ((IOC_NET_HEADER_FIELD_IPv6_VER << 7) - 1) + +#define IOC_NET_HEADER_FIELD_IPv6_ADDR_SIZE 16 +#define IOC_NET_HEADER_FIELD_IPv6_NEXT_HDR_SIZE 1 + +#define IOC_NET_HEADER_FIELD_ICMP_TYPE (1) +#define IOC_NET_HEADER_FIELD_ICMP_CODE (IOC_NET_HEADER_FIELD_ICMP_TYPE << 1) +#define IOC_NET_HEADER_FIELD_ICMP_CKSUM (IOC_NET_HEADER_FIELD_ICMP_TYPE << 2) +#define IOC_NET_HEADER_FIELD_ICMP_ID (IOC_NET_HEADER_FIELD_ICMP_TYPE << 3) +#define IOC_NET_HEADER_FIELD_ICMP_SQ_NUM (IOC_NET_HEADER_FIELD_ICMP_TYPE << 4) +#define IOC_NET_HEADER_FIELD_ICMP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_ICMP_TYPE << 5) - 1) + +#define IOC_NET_HEADER_FIELD_ICMP_CODE_SIZE 1 +#define IOC_NET_HEADER_FIELD_ICMP_TYPE_SIZE 1 + +#define IOC_NET_HEADER_FIELD_IGMP_VERSION (1) +#define IOC_NET_HEADER_FIELD_IGMP_TYPE (IOC_NET_HEADER_FIELD_IGMP_VERSION << 1) +#define IOC_NET_HEADER_FIELD_IGMP_CKSUM (IOC_NET_HEADER_FIELD_IGMP_VERSION << 2) +#define IOC_NET_HEADER_FIELD_IGMP_DATA (IOC_NET_HEADER_FIELD_IGMP_VERSION << 3) +#define IOC_NET_HEADER_FIELD_IGMP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_IGMP_VERSION << 4) - 1) + + +typedef uint16_t ioc_header_field_tcp_t; + +#define IOC_NET_HEADER_FIELD_TCP_PORT_SRC (1) +#define IOC_NET_HEADER_FIELD_TCP_PORT_DST (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 1) +#define IOC_NET_HEADER_FIELD_TCP_SEQ (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 2) +#define IOC_NET_HEADER_FIELD_TCP_ACK (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 3) +#define IOC_NET_HEADER_FIELD_TCP_OFFSET (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 4) +#define IOC_NET_HEADER_FIELD_TCP_FLAGS (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 5) +#define IOC_NET_HEADER_FIELD_TCP_WINDOW (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 6) +#define IOC_NET_HEADER_FIELD_TCP_CKSUM (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 7) +#define IOC_NET_HEADER_FIELD_TCP_URGPTR (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 8) +#define IOC_NET_HEADER_FIELD_TCP_OPTS (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 9) +#define IOC_NET_HEADER_FIELD_TCP_OPTS_COUNT (IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 10) +#define IOC_NET_HEADER_FIELD_TCP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_TCP_PORT_SRC << 11) - 1) + +#define IOC_NET_HEADER_FIELD_TCP_PORT_SIZE 2 + + +typedef uint8_t ioc_header_field_sctp_t; + +#define IOC_NET_HEADER_FIELD_SCTP_PORT_SRC (1) +#define IOC_NET_HEADER_FIELD_SCTP_PORT_DST (IOC_NET_HEADER_FIELD_SCTP_PORT_SRC << 1) +#define IOC_NET_HEADER_FIELD_SCTP_VER_TAG (IOC_NET_HEADER_FIELD_SCTP_PORT_SRC << 2) +#define IOC_NET_HEADER_FIELD_SCTP_CKSUM (IOC_NET_HEADER_FIELD_SCTP_PORT_SRC << 3) +#define IOC_NET_HEADER_FIELD_SCTP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_SCTP_PORT_SRC << 4) - 1) + +#define IOC_NET_HEADER_FIELD_SCTP_PORT_SIZE 2 + +typedef uint8_t ioc_header_field_dccp_t; + +#define IOC_NET_HEADER_FIELD_DCCP_PORT_SRC (1) +#define IOC_NET_HEADER_FIELD_DCCP_PORT_DST (IOC_NET_HEADER_FIELD_DCCP_PORT_SRC << 1) +#define IOC_NET_HEADER_FIELD_DCCP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_DCCP_PORT_SRC << 2) - 1) + +#define IOC_NET_HEADER_FIELD_DCCP_PORT_SIZE 2 + + +typedef uint8_t ioc_header_field_udp_t; + +#define IOC_NET_HEADER_FIELD_UDP_PORT_SRC (1) +#define IOC_NET_HEADER_FIELD_UDP_PORT_DST (IOC_NET_HEADER_FIELD_UDP_PORT_SRC << 1) +#define IOC_NET_HEADER_FIELD_UDP_LEN (IOC_NET_HEADER_FIELD_UDP_PORT_SRC << 2) +#define IOC_NET_HEADER_FIELD_UDP_CKSUM (IOC_NET_HEADER_FIELD_UDP_PORT_SRC << 3) +#define IOC_NET_HEADER_FIELD_UDP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_UDP_PORT_SRC << 4) - 1) + +#define IOC_NET_HEADER_FIELD_UDP_PORT_SIZE 2 + +typedef uint8_t ioc_header_field_udp_lite_t; + +#define IOC_NET_HEADER_FIELD_UDP_LITE_PORT_SRC (1) +#define IOC_NET_HEADER_FIELD_UDP_LITE_PORT_DST (IOC_NET_HEADER_FIELD_UDP_LITE_PORT_SRC << 1) +#define IOC_NET_HEADER_FIELD_UDP_LITE_ALL_FIELDS ((IOC_NET_HEADER_FIELD_UDP_LITE_PORT_SRC << 2) - 1) + +#define IOC_NET_HEADER_FIELD_UDP_LITE_PORT_SIZE 2 + +typedef uint8_t ioc_header_field_udp_encap_esp_t; + +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_SRC (1) +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_DST (IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_SRC << 1) +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_LEN (IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_SRC << 2) +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_CKSUM (IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_SRC << 3) +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_SPI (IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_SRC << 4) +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_SEQUENCE_NUM (IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_SRC << 5) +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_SRC << 6) - 1) + +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_PORT_SIZE 2 +#define IOC_NET_HEADER_FIELD_UDP_ENCAP_ESP_SPI_SIZE 4 + +#define IOC_NET_HEADER_FIELD_IPHC_CID (1) +#define IOC_NET_HEADER_FIELD_IPHC_CID_TYPE (IOC_NET_HEADER_FIELD_IPHC_CID << 1) +#define IOC_NET_HEADER_FIELD_IPHC_HCINDEX (IOC_NET_HEADER_FIELD_IPHC_CID << 2) +#define IOC_NET_HEADER_FIELD_IPHC_GEN (IOC_NET_HEADER_FIELD_IPHC_CID << 3) +#define IOC_NET_HEADER_FIELD_IPHC_D_BIT (IOC_NET_HEADER_FIELD_IPHC_CID << 4) +#define IOC_NET_HEADER_FIELD_IPHC_ALL_FIELDS ((IOC_NET_HEADER_FIELD_IPHC_CID << 5) - 1) + +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE (1) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_FLAGS (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 1) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_LENGTH (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 2) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TSN (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 3) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_STREAM_ID (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 4) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_STREAM_SQN (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 5) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_PAYLOAD_PID (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 6) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_UNORDERED (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 7) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_BEGGINING (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 8) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_END (IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 9) +#define IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_ALL_FIELDS ((IOC_NET_HEADER_FIELD_SCTP_CHUNK_DATA_TYPE << 10) - 1) + +#define IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT (1) +#define IOC_NET_HEADER_FIELD_L2TPv2_LENGTH_BIT (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 1) +#define IOC_NET_HEADER_FIELD_L2TPv2_SEQUENCE_BIT (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 2) +#define IOC_NET_HEADER_FIELD_L2TPv2_OFFSET_BIT (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 3) +#define IOC_NET_HEADER_FIELD_L2TPv2_PRIORITY_BIT (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 4) +#define IOC_NET_HEADER_FIELD_L2TPv2_VERSION (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 5) +#define IOC_NET_HEADER_FIELD_L2TPv2_LEN (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 6) +#define IOC_NET_HEADER_FIELD_L2TPv2_TUNNEL_ID (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 7) +#define IOC_NET_HEADER_FIELD_L2TPv2_SESSION_ID (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 8) +#define IOC_NET_HEADER_FIELD_L2TPv2_NS (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 9) +#define IOC_NET_HEADER_FIELD_L2TPv2_NR (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 10) +#define IOC_NET_HEADER_FIELD_L2TPv2_OFFSET_SIZE (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 11) +#define IOC_NET_HEADER_FIELD_L2TPv2_FIRST_BYTE (IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 12) +#define IOC_NET_HEADER_FIELD_L2TPv2_ALL_FIELDS ((IOC_NET_HEADER_FIELD_L2TPv2_TYPE_BIT << 13) - 1) + +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT (1) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_LENGTH_BIT (IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 1) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_SEQUENCE_BIT (IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 2) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_VERSION (IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 3) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_LENGTH (IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 4) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_CONTROL (IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 5) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_SENT (IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 6) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_RECV (IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 7) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_FIRST_BYTE (IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 8) +#define IOC_NET_HEADER_FIELD_L2TPv3_CTRL_ALL_FIELDS ((IOC_NET_HEADER_FIELD_L2TPv3_CTRL_TYPE_BIT << 9) - 1) + +#define IOC_NET_HEADER_FIELD_L2TPv3_SESS_TYPE_BIT (1) +#define IOC_NET_HEADER_FIELD_L2TPv3_SESS_VERSION (IOC_NET_HEADER_FIELD_L2TPv3_SESS_TYPE_BIT << 1) +#define IOC_NET_HEADER_FIELD_L2TPv3_SESS_ID (IOC_NET_HEADER_FIELD_L2TPv3_SESS_TYPE_BIT << 2) +#define IOC_NET_HEADER_FIELD_L2TPv3_SESS_COOKIE (IOC_NET_HEADER_FIELD_L2TPv3_SESS_TYPE_BIT << 3) +#define IOC_NET_HEADER_FIELD_L2TPv3_SESS_ALL_FIELDS ((IOC_NET_HEADER_FIELD_L2TPv3_SESS_TYPE_BIT << 4) - 1) + + +typedef uint8_t ioc_header_field_vlan_t; + +#define IOC_NET_HEADER_FIELD_VLAN_VPRI (1) +#define IOC_NET_HEADER_FIELD_VLAN_CFI (IOC_NET_HEADER_FIELD_VLAN_VPRI << 1) +#define IOC_NET_HEADER_FIELD_VLAN_VID (IOC_NET_HEADER_FIELD_VLAN_VPRI << 2) +#define IOC_NET_HEADER_FIELD_VLAN_LENGTH (IOC_NET_HEADER_FIELD_VLAN_VPRI << 3) +#define IOC_NET_HEADER_FIELD_VLAN_TYPE (IOC_NET_HEADER_FIELD_VLAN_VPRI << 4) +#define IOC_NET_HEADER_FIELD_VLAN_ALL_FIELDS ((IOC_NET_HEADER_FIELD_VLAN_VPRI << 5) - 1) + +#define IOC_NET_HEADER_FIELD_VLAN_TCI (IOC_NET_HEADER_FIELD_VLAN_VPRI | \ + IOC_NET_HEADER_FIELD_VLAN_CFI | \ + IOC_NET_HEADER_FIELD_VLAN_VID) + + +typedef uint8_t ioc_header_field_llc_t; + +#define IOC_NET_HEADER_FIELD_LLC_DSAP (1) +#define IOC_NET_HEADER_FIELD_LLC_SSAP (IOC_NET_HEADER_FIELD_LLC_DSAP << 1) +#define IOC_NET_HEADER_FIELD_LLC_CTRL (IOC_NET_HEADER_FIELD_LLC_DSAP << 2) +#define IOC_NET_HEADER_FIELD_LLC_ALL_FIELDS ((IOC_NET_HEADER_FIELD_LLC_DSAP << 3) - 1) + +#define IOC_NET_HEADER_FIELD_NLPID_NLPID (1) +#define IOC_NET_HEADER_FIELD_NLPID_ALL_FIELDS ((IOC_NET_HEADER_FIELD_NLPID_NLPID << 1) - 1) + + +typedef uint8_t ioc_header_field_snap_t; + +#define IOC_NET_HEADER_FIELD_SNAP_OUI (1) +#define IOC_NET_HEADER_FIELD_SNAP_PID (IOC_NET_HEADER_FIELD_SNAP_OUI << 1) +#define IOC_NET_HEADER_FIELD_SNAP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_SNAP_OUI << 2) - 1) + + +typedef uint8_t ioc_header_field_llc_snap_t; + +#define IOC_NET_HEADER_FIELD_LLC_SNAP_TYPE (1) +#define IOC_NET_HEADER_FIELD_LLC_SNAP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_LLC_SNAP_TYPE << 1) - 1) + +#define IOC_NET_HEADER_FIELD_ARP_HTYPE (1) +#define IOC_NET_HEADER_FIELD_ARP_PTYPE (IOC_NET_HEADER_FIELD_ARP_HTYPE << 1) +#define IOC_NET_HEADER_FIELD_ARP_HLEN (IOC_NET_HEADER_FIELD_ARP_HTYPE << 2) +#define IOC_NET_HEADER_FIELD_ARP_PLEN (IOC_NET_HEADER_FIELD_ARP_HTYPE << 3) +#define IOC_NET_HEADER_FIELD_ARP_OPER (IOC_NET_HEADER_FIELD_ARP_HTYPE << 4) +#define IOC_NET_HEADER_FIELD_ARP_SHA (IOC_NET_HEADER_FIELD_ARP_HTYPE << 5) +#define IOC_NET_HEADER_FIELD_ARP_SPA (IOC_NET_HEADER_FIELD_ARP_HTYPE << 6) +#define IOC_NET_HEADER_FIELD_ARP_THA (IOC_NET_HEADER_FIELD_ARP_HTYPE << 7) +#define IOC_NET_HEADER_FIELD_ARP_TPA (IOC_NET_HEADER_FIELD_ARP_HTYPE << 8) +#define IOC_NET_HEADER_FIELD_ARP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_ARP_HTYPE << 9) - 1) + +#define IOC_NET_HEADER_FIELD_RFC2684_LLC (1) +#define IOC_NET_HEADER_FIELD_RFC2684_NLPID (IOC_NET_HEADER_FIELD_RFC2684_LLC << 1) +#define IOC_NET_HEADER_FIELD_RFC2684_OUI (IOC_NET_HEADER_FIELD_RFC2684_LLC << 2) +#define IOC_NET_HEADER_FIELD_RFC2684_PID (IOC_NET_HEADER_FIELD_RFC2684_LLC << 3) +#define IOC_NET_HEADER_FIELD_RFC2684_VPN_OUI (IOC_NET_HEADER_FIELD_RFC2684_LLC << 4) +#define IOC_NET_HEADER_FIELD_RFC2684_VPN_IDX (IOC_NET_HEADER_FIELD_RFC2684_LLC << 5) +#define IOC_NET_HEADER_FIELD_RFC2684_ALL_FIELDS ((IOC_NET_HEADER_FIELD_RFC2684_LLC << 6) - 1) + +#define IOC_NET_HEADER_FIELD_USER_DEFINED_SRCPORT (1) +#define IOC_NET_HEADER_FIELD_USER_DEFINED_PCDID (IOC_NET_HEADER_FIELD_USER_DEFINED_SRCPORT << 1) +#define IOC_NET_HEADER_FIELD_USER_DEFINED_ALL_FIELDS ((IOC_NET_HEADER_FIELD_USER_DEFINED_SRCPORT << 2) - 1) + +#define IOC_NET_HEADER_FIELD_PAYLOAD_BUFFER (1) +#define IOC_NET_HEADER_FIELD_PAYLOAD_SIZE (IOC_NET_HEADER_FIELD_PAYLOAD_BUFFER << 1) +#define IOC_NET_HEADER_FIELD_MAX_FRM_SIZE (IOC_NET_HEADER_FIELD_PAYLOAD_BUFFER << 2) +#define IOC_NET_HEADER_FIELD_MIN_FRM_SIZE (IOC_NET_HEADER_FIELD_PAYLOAD_BUFFER << 3) +#define IOC_NET_HEADER_FIELD_PAYLOAD_TYPE (IOC_NET_HEADER_FIELD_PAYLOAD_BUFFER << 4) +#define IOC_NET_HEADER_FIELD_FRAME_SIZE (IOC_NET_HEADER_FIELD_PAYLOAD_BUFFER << 5) +#define IOC_NET_HEADER_FIELD_PAYLOAD_ALL_FIELDS ((IOC_NET_HEADER_FIELD_PAYLOAD_BUFFER << 6) - 1) + + +typedef uint8_t ioc_header_field_gre_t; + +#define IOC_NET_HEADER_FIELD_GRE_TYPE (1) +#define IOC_NET_HEADER_FIELD_GRE_ALL_FIELDS ((IOC_NET_HEADER_FIELD_GRE_TYPE << 1) - 1) + + +typedef uint8_t ioc_header_field_minencap_t; + +#define IOC_NET_HEADER_FIELD_MINENCAP_SRC_IP (1) +#define IOC_NET_HEADER_FIELD_MINENCAP_DST_IP (IOC_NET_HEADER_FIELD_MINENCAP_SRC_IP << 1) +#define IOC_NET_HEADER_FIELD_MINENCAP_TYPE (IOC_NET_HEADER_FIELD_MINENCAP_SRC_IP << 2) +#define IOC_NET_HEADER_FIELD_MINENCAP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_MINENCAP_SRC_IP << 3) - 1) + + +typedef uint8_t ioc_header_field_ipsec_ah_t; + +#define IOC_NET_HEADER_FIELD_IPSEC_AH_SPI (1) +#define IOC_NET_HEADER_FIELD_IPSEC_AH_NH (IOC_NET_HEADER_FIELD_IPSEC_AH_SPI << 1) +#define IOC_NET_HEADER_FIELD_IPSEC_AH_ALL_FIELDS ((IOC_NET_HEADER_FIELD_IPSEC_AH_SPI << 2) - 1) + + +typedef uint8_t ioc_header_field_ipsec_esp_t; + +#define IOC_NET_HEADER_FIELD_IPSEC_ESP_SPI (1) +#define IOC_NET_HEADER_FIELD_IPSEC_ESP_SEQUENCE_NUM (IOC_NET_HEADER_FIELD_IPSEC_ESP_SPI << 1) +#define IOC_NET_HEADER_FIELD_IPSEC_ESP_ALL_FIELDS ((IOC_NET_HEADER_FIELD_IPSEC_ESP_SPI << 2) - 1) + +#define IOC_NET_HEADER_FIELD_IPSEC_ESP_SPI_SIZE 4 + + +typedef uint8_t ioc_header_field_mpls_t; + +#define IOC_NET_HEADER_FIELD_MPLS_LABEL_STACK (1) +#define IOC_NET_HEADER_FIELD_MPLS_LABEL_STACK_ALL_FIELDS ((IOC_NET_HEADER_FIELD_MPLS_LABEL_STACK << 1) - 1) + + +typedef uint8_t ioc_header_field_macsec_t; + +#define IOC_NET_HEADER_FIELD_MACSEC_SECTAG (1) +#define IOC_NET_HEADER_FIELD_MACSEC_ALL_FIELDS ((IOC_NET_HEADER_FIELD_MACSEC_SECTAG << 1) - 1) + + +typedef enum { + e_IOC_NET_HEADER_TYPE_NONE = 0, + e_IOC_NET_HEADER_TYPE_PAYLOAD, + e_IOC_NET_HEADER_TYPE_ETH, + e_IOC_NET_HEADER_TYPE_VLAN, + e_IOC_NET_HEADER_TYPE_IPv4, + e_IOC_NET_HEADER_TYPE_IPv6, + e_IOC_NET_HEADER_TYPE_IP, + e_IOC_NET_HEADER_TYPE_TCP, + e_IOC_NET_HEADER_TYPE_UDP, + e_IOC_NET_HEADER_TYPE_UDP_LITE, + e_IOC_NET_HEADER_TYPE_IPHC, + e_IOC_NET_HEADER_TYPE_SCTP, + e_IOC_NET_HEADER_TYPE_SCTP_CHUNK_DATA, + e_IOC_NET_HEADER_TYPE_PPPoE, + e_IOC_NET_HEADER_TYPE_PPP, + e_IOC_NET_HEADER_TYPE_PPPMUX, + e_IOC_NET_HEADER_TYPE_PPPMUX_SUBFRAME, + e_IOC_NET_HEADER_TYPE_L2TPv2, + e_IOC_NET_HEADER_TYPE_L2TPv3_CTRL, + e_IOC_NET_HEADER_TYPE_L2TPv3_SESS, + e_IOC_NET_HEADER_TYPE_LLC, + e_IOC_NET_HEADER_TYPE_LLC_SNAP, + e_IOC_NET_HEADER_TYPE_NLPID, + e_IOC_NET_HEADER_TYPE_SNAP, + e_IOC_NET_HEADER_TYPE_MPLS, + e_IOC_NET_HEADER_TYPE_IPSEC_AH, + e_IOC_NET_HEADER_TYPE_IPSEC_ESP, + e_IOC_NET_HEADER_TYPE_UDP_ENCAP_ESP, /* RFC 3948 */ + e_IOC_NET_HEADER_TYPE_MACSEC, + e_IOC_NET_HEADER_TYPE_GRE, + e_IOC_NET_HEADER_TYPE_MINENCAP, + e_IOC_NET_HEADER_TYPE_DCCP, + e_IOC_NET_HEADER_TYPE_ICMP, + e_IOC_NET_HEADER_TYPE_IGMP, + e_IOC_NET_HEADER_TYPE_ARP, + e_IOC_NET_HEADER_TYPE_CAPWAP, + e_IOC_NET_HEADER_TYPE_CAPWAP_DTLS, + e_IOC_NET_HEADER_TYPE_RFC2684, + e_IOC_NET_HEADER_TYPE_USER_DEFINED_L2, + e_IOC_NET_HEADER_TYPE_USER_DEFINED_L3, + e_IOC_NET_HEADER_TYPE_USER_DEFINED_L4, + e_IOC_NET_HEADER_TYPE_USER_DEFINED_SHIM1, + e_IOC_NET_HEADER_TYPE_USER_DEFINED_SHIM2, + e_IOC_NET_MAX_HEADER_TYPE_COUNT +} ioc_net_header_type; + + +#endif /* __NET_IOCTLS_H */ diff --git a/sys/contrib/ncsw/inc/types_freebsd.h b/sys/contrib/ncsw/inc/types_freebsd.h index 031da08e0f75..e37fb2502c24 100644 --- a/sys/contrib/ncsw/inc/types_freebsd.h +++ b/sys/contrib/ncsw/inc/types_freebsd.h @@ -36,7 +36,7 @@ #include #if !defined(__bool_true_false_are_defined) -typedef boolean_t bool; +typedef _Bool bool; #endif #define TRUE 1 #define FALSE 0 @@ -48,22 +48,39 @@ typedef vm_paddr_t physAddress_t; /** * Accessor defines. - * TODO: These are only stubs and have to be redefined (use bus_space - * facilities). + * DPAA1 registers are big-endian. On PowerPC (big-endian), native + * in32/out32 match. On ARM64 (little-endian), use the reverse-byte + * variants (in32rb/out32rb) to byte-swap. */ +#ifdef __aarch64__ +#define GET_UINT32(arg) in32rb(&(arg)) +#define GET_UINT64(arg) in64(&(arg)) + +#define _WRITE_UINT32(arg, data) out32rb(&(arg), (data)) +#define _WRITE_UINT64(arg, data) out64(&(arg), (data)) +#else #define GET_UINT32(arg) in32(&(arg)) #define GET_UINT64(arg) in64(&(arg)) #define _WRITE_UINT32(arg, data) out32(&(arg), (data)) #define _WRITE_UINT64(arg, data) out64(&(arg), (data)) +#endif #ifndef QE_32_BIT_ACCESS_RESTRICTION +#ifdef __aarch64__ +#define GET_UINT8(arg) in8(&(arg)) +#define GET_UINT16(arg) in16rb(&(arg)) + +#define _WRITE_UINT8(arg, data) out8(&(arg), (data)) +#define _WRITE_UINT16(arg, data) out16rb(&(arg), (data)) +#else #define GET_UINT8(arg) in8(&(arg)) #define GET_UINT16(arg) in16(&(arg)) #define _WRITE_UINT8(arg, data) out8(&(arg), (data)) #define _WRITE_UINT16(arg, data) out16(&(arg), (data)) +#endif #else /* QE_32_BIT_ACCESS_RESTRICTION */ diff --git a/sys/contrib/ncsw/user/env/core_arm64.c b/sys/contrib/ncsw/user/env/core_arm64.c new file mode 100644 index 000000000000..552d6be43caa --- /dev/null +++ b/sys/contrib/ncsw/user/env/core_arm64.c @@ -0,0 +1,60 @@ +/*- + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * ARM64 core functions for NetCommSW (DPAA1). + * Equivalent to core.c (which provides E500_GetId for PowerPC). + */ +#include +#include +#include +#include + +#include + +#include "types_ext.h" +#include "core_ext.h" + +uint32_t +ARM64_GetId(void) +{ + + return (PCPU_GET(cpuid)); +} + +int +ARM64_TestAndSet(volatile int *p) +{ + + return (!atomic_cmpset_int((volatile u_int *)p, 0, 1)); +} + +void +ARM64_InstructionSync(void) +{ + + __asm __volatile("isb" ::: "memory"); +} diff --git a/sys/contrib/ncsw/user/env/xx_arm64.c b/sys/contrib/ncsw/user/env/xx_arm64.c new file mode 100644 index 000000000000..25ece0021f50 --- /dev/null +++ b/sys/contrib/ncsw/user/env/xx_arm64.c @@ -0,0 +1,797 @@ +/*- + * Copyright (c) 2011 Semihalf. + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * NetCommSW (ncsw) environment layer for FreeBSD ARM64. + * + * Adapted from xx.c (PowerPC version) by Semihalf. Key differences: + * + * 1. No ccsrbar_va/pa (ARM64 uses FDT-based bus_space mappings) + * 2. No law_enable (ARM64 doesn't have Local Access Windows) + * 3. No powerpc_intr_mask/unmask (GIC handles interrupt masking) + * 4. No TLB1 management (ARM64 uses page tables) + * 5. XX_VirtToPhys/XX_PhysToVirt use DMAP exclusively + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "error_ext.h" +#include "std_ext.h" +#include "list_ext.h" +#include "mm_ext.h" + +/* Malloc Pool for NetCommSW */ +MALLOC_DEFINE(M_NETCOMMSW, "NetCommSW", "NetCommSW software stack"); +MALLOC_DEFINE(M_NETCOMMSW_MT, "NetCommSWTrack", + "NetCommSW software allocation tracker"); + +static struct mtx XX_MallocTrackLock; +MTX_SYSINIT(XX_MallocTrackLockInit, &XX_MallocTrackLock, + "NetCommSW MallocTrack Lock", MTX_DEF); + +/* Interrupt info */ +#define XX_INTR_FLAG_PREALLOCATED (1 << 0) + +/* + * ARM64 GIC can have many more interrupts than PowerPC (INTR_VECTORS=256). + * DPAA1 only uses a handful; 512 is plenty. + */ +#define XX_MAX_INTR 512 + +struct XX_IntrInfo { + driver_intr_t *handler; + void *arg; + int cpu; + int flags; + void *cookie; + /* NAPI mode: non-NULL napi_tq enables FILTER handler */ + struct taskqueue *napi_tq; + struct task *napi_task; + t_Handle napi_portal; +}; + +static struct XX_IntrInfo XX_IntrInfo[XX_MAX_INTR]; + +/* Portal type identifiers */ +enum XX_PortalIdent{ + BM_PORTAL = 0, + QM_PORTAL, +}; + +/* Structure to store portals' properties */ +struct XX_PortalInfo { + vm_paddr_t portal_ce_pa[2][MAXCPU]; + vm_paddr_t portal_ci_pa[2][MAXCPU]; + uint32_t portal_ce_size[2][MAXCPU]; + uint32_t portal_ci_size[2][MAXCPU]; + vm_offset_t portal_ce_va[2]; + vm_offset_t portal_ci_va[2]; + uintptr_t portal_intr[2][MAXCPU]; +}; + +static struct XX_PortalInfo XX_PInfo; + +void +XX_Exit(int status) +{ + + panic("NetCommSW: Exit called with status %i", status); +} + +void +XX_Print(char *str, ...) +{ + va_list ap; + + va_start(ap, str); + vprintf(str, ap); + va_end(ap); +} + +void * +XX_Malloc(uint32_t size) +{ + void *p = (malloc(size, M_NETCOMMSW, M_NOWAIT)); + + return (p); +} + +/* + * XX_MallocSmartInit — no-op, kept for API compatibility. + * Previously allocated a fixed contigmalloc pool; now XX_MallocSmart + * delegates to XX_Malloc (kernel heap) with manual alignment, matching + * the Linux NCSW implementation. + */ +int +XX_MallocSmartInit(void) +{ + + return (E_OK); +} + +/* + * XX_MallocSmart — aligned allocation via kernel heap. + * + * Matches Linux NCSW xx_arm_linux.c: allocate size + alignment + header, + * align the returned pointer, store the original malloc pointer in the + * word immediately before it so XX_FreeSmart can recover it. + */ +void * +XX_MallocSmart(uint32_t size, int memPartitionId, uint32_t alignment) +{ + uintptr_t *aligned, raw; + + if (alignment < sizeof(uintptr_t)) + alignment = sizeof(uintptr_t); + + size += alignment + sizeof(uintptr_t); + raw = (uintptr_t)XX_Malloc(size); + if (raw == 0) + return (NULL); + + aligned = (uintptr_t *)((raw + alignment + sizeof(uintptr_t)) + & ~((uintptr_t)alignment - 1)); + *(aligned - 1) = raw; + + return ((void *)aligned); +} + +void +XX_FreeSmart(void *p) +{ + + if (p != NULL) + XX_Free((void *)(*(((uintptr_t *)p) - 1))); +} + +void +XX_Free(void *p) +{ + + free(p, M_NETCOMMSW); +} + +uint32_t +XX_DisableAllIntr(void) +{ + + return (intr_disable()); +} + +void +XX_RestoreAllIntr(uint32_t flags) +{ + + intr_restore(flags); +} + +static bool +XX_IsPortalIntr(uintptr_t irq) +{ + int cpu, type; + /* Check interrupt numbers of all available portals */ + for (type = 0; type < 2; type++) + for (cpu = 0; cpu < MAXCPU; cpu++) + if (irq == XX_PInfo.portal_intr[type][cpu]) + return (1); + + return (0); +} + +static void +XX_Dispatch(void *arg) +{ + struct XX_IntrInfo *info; + + info = arg; + + if (info->handler == NULL) { + printf("%s(): IRQ handler is NULL!\n", __func__); + return; + } + + info->handler(info->arg); +} + +/* + * FILTER interrupt handler for NAPI-style portal processing. + * Runs in hard IRQ context — inhibits portal interrupts at the + * source (IIR=1) and schedules a taskqueue task for deferred + * DQRR/MR drain. Matches Linux's portal_isr → napi_schedule pattern. + */ +static int +XX_PortalFilter(void *arg) +{ + struct XX_IntrInfo *info = arg; + + if (__predict_false(info->napi_portal == NULL)) + return (FILTER_STRAY); + + QM_PORTAL_Inhibit(info->napi_portal); + taskqueue_enqueue(info->napi_tq, info->napi_task); + + return (FILTER_HANDLED); +} + +/* + * Configure NAPI mode for a portal interrupt. Must be called + * before QM_PORTAL_Init (which calls XX_SetIntr internally). + */ +void XX_ConfigPortalNapi(uintptr_t irq, struct taskqueue *tq, struct task *task); +void XX_SetPortalNapiHandle(uintptr_t irq, t_Handle portal); +void +XX_ConfigPortalNapi(uintptr_t irq, struct taskqueue *tq, struct task *task) +{ + struct resource *r; + unsigned int inum; + + r = (struct resource *)irq; + inum = rman_get_start(r); + + if (inum >= XX_MAX_INTR) + return; + + XX_IntrInfo[inum].napi_tq = tq; + XX_IntrInfo[inum].napi_task = task; + XX_IntrInfo[inum].napi_portal = NULL; +} + +/* + * Set the portal handle for a NAPI-configured interrupt. + * Called after QM_PORTAL_Init returns the portal handle. + */ +void +XX_SetPortalNapiHandle(uintptr_t irq, t_Handle portal) +{ + struct resource *r; + unsigned int inum; + + r = (struct resource *)irq; + inum = rman_get_start(r); + + if (inum >= XX_MAX_INTR) + return; + + XX_IntrInfo[inum].napi_portal = portal; +} + +t_Error +XX_PreallocAndBindIntr(device_t dev, uintptr_t irq, unsigned int cpu) +{ + struct resource *r; + unsigned int inum; + + r = (struct resource *)irq; + inum = rman_get_start(r); + + if (inum >= XX_MAX_INTR) { + printf("NetCommSW: IRQ %u exceeds XX_MAX_INTR (%d)\n", + inum, XX_MAX_INTR); + return (E_FULL); + } + + /* + * Don't call bus_setup_intr here — just record the CPU binding. + * The OS interrupt handler will be registered later when the + * portal driver calls XX_SetIntr with a real handler. This + * prevents interrupt storms from stale portal state on warm + * reboot (portal hardware may assert IRQ before the driver + * is ready to clear it). + */ + XX_IntrInfo[inum].flags = XX_INTR_FLAG_PREALLOCATED; + XX_IntrInfo[inum].cpu = cpu; + + return (E_OK); +} + +t_Error +XX_DeallocIntr(uintptr_t irq) +{ + struct resource *r; + unsigned int inum; + + r = (struct resource *)irq; + inum = rman_get_start(r); + + if (inum >= XX_MAX_INTR) + return (E_INVALID_STATE); + + if ((XX_IntrInfo[inum].flags & XX_INTR_FLAG_PREALLOCATED) == 0) + return (E_INVALID_STATE); + + XX_IntrInfo[inum].flags = 0; + XX_IntrInfo[inum].handler = NULL; + XX_IntrInfo[inum].arg = NULL; + + /* Tear down OS handler only if it was registered */ + if (XX_IntrInfo[inum].cookie != NULL) + return (XX_FreeIntr(irq)); + + return (E_OK); +} + +t_Error +XX_SetIntr(uintptr_t irq, t_Isr *f_Isr, t_Handle handle) +{ + device_t dev; + struct resource *r; + unsigned int flags; + int err; + + r = (struct resource *)irq; + dev = rman_get_device(r); + irq = rman_get_start(r); + + if (irq >= XX_MAX_INTR) + return (E_FULL); + + /* Handle preallocated interrupts */ + if (XX_IntrInfo[irq].flags & XX_INTR_FLAG_PREALLOCATED) { + if (XX_IntrInfo[irq].handler != NULL) + return (E_BUSY); + + XX_IntrInfo[irq].handler = f_Isr; + XX_IntrInfo[irq].arg = handle; + + /* Register OS handler now that portal driver is ready */ + if (XX_IntrInfo[irq].cookie == NULL) { + unsigned int iflags; + int err; + + iflags = INTR_TYPE_NET | INTR_MPSAFE; + if (XX_IsPortalIntr(irq)) + iflags |= INTR_EXCL; + + if (XX_IntrInfo[irq].napi_tq != NULL) { + /* NAPI mode: FILTER-only, no ithread */ + err = bus_setup_intr(dev, r, iflags, + XX_PortalFilter, NULL, + &XX_IntrInfo[irq], + &XX_IntrInfo[irq].cookie); + } else { + /* Legacy ithread mode */ + err = bus_setup_intr(dev, r, iflags, NULL, + XX_Dispatch, &XX_IntrInfo[irq], + &XX_IntrInfo[irq].cookie); + } + if (err) + return (E_NOT_AVAILABLE); + + bus_bind_intr(dev, r, XX_IntrInfo[irq].cpu); + } + + return (E_OK); + } + + flags = INTR_TYPE_NET | INTR_MPSAFE; + + /* BMAN/QMAN Portal interrupts must be exclusive */ + if (XX_IsPortalIntr(irq)) + flags |= INTR_EXCL; + + err = bus_setup_intr(dev, r, flags, NULL, f_Isr, handle, + &XX_IntrInfo[irq].cookie); + + return (err); +} + +t_Error +XX_FreeIntr(uintptr_t irq) +{ + device_t dev; + struct resource *r; + + r = (struct resource *)irq; + dev = rman_get_device(r); + irq = rman_get_start(r); + + if (irq >= XX_MAX_INTR) + return (E_INVALID_STATE); + + /* Handle preallocated interrupts */ + if (XX_IntrInfo[irq].flags & XX_INTR_FLAG_PREALLOCATED) { + if (XX_IntrInfo[irq].handler == NULL) + return (E_INVALID_STATE); + + XX_IntrInfo[irq].handler = NULL; + XX_IntrInfo[irq].arg = NULL; + + return (E_OK); + } + + return (bus_teardown_intr(dev, r, XX_IntrInfo[irq].cookie)); +} + +t_Error +XX_EnableIntr(uintptr_t irq) +{ + /* + * On ARM64 with GIC, interrupt masking/unmasking is handled + * by the interrupt framework. Portal interrupts are set up + * with INTR_EXCL; the GIC re-enables them after handler return. + */ + (void)irq; + return (E_OK); +} + +t_Error +XX_DisableIntr(uintptr_t irq) +{ + /* + * See XX_EnableIntr — no explicit mask needed on ARM64. + */ + (void)irq; + return (E_OK); +} + +t_TaskletHandle +XX_InitTasklet (void (*routine)(void *), void *data) +{ + /* Not referenced */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); + return (NULL); +} + + +void +XX_FreeTasklet (t_TaskletHandle h_Tasklet) +{ + /* Not referenced */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); +} + +int +XX_ScheduleTask(t_TaskletHandle h_Tasklet, int immediate) +{ + /* Not referenced */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); + return (0); +} + +void +XX_FlushScheduledTasks(void) +{ + /* Not referenced */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); +} + +int +XX_TaskletIsQueued(t_TaskletHandle h_Tasklet) +{ + /* Not referenced */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); + return (0); +} + +void +XX_SetTaskletData(t_TaskletHandle h_Tasklet, t_Handle data) +{ + /* Not referenced */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); +} + +t_Handle +XX_GetTaskletData(t_TaskletHandle h_Tasklet) +{ + /* Not referenced */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); + return (NULL); +} + +t_Handle +XX_InitSpinlock(void) +{ + struct mtx *m; + + m = malloc(sizeof(*m), M_NETCOMMSW, M_NOWAIT | M_ZERO); + if (!m) + return (0); + + mtx_init(m, "NetCommSW Lock", NULL, MTX_DEF | MTX_DUPOK); + + return (m); +} + +void +XX_FreeSpinlock(t_Handle h_Spinlock) +{ + struct mtx *m; + + m = h_Spinlock; + + mtx_destroy(m); + free(m, M_NETCOMMSW); +} + +void +XX_LockSpinlock(t_Handle h_Spinlock) +{ + struct mtx *m; + + m = h_Spinlock; + mtx_lock(m); +} + +void +XX_UnlockSpinlock(t_Handle h_Spinlock) +{ + struct mtx *m; + + m = h_Spinlock; + mtx_unlock(m); +} + +uint32_t +XX_LockIntrSpinlock(t_Handle h_Spinlock) +{ + + XX_LockSpinlock(h_Spinlock); + return (0); +} + +void +XX_UnlockIntrSpinlock(t_Handle h_Spinlock, uint32_t intrFlags) +{ + + XX_UnlockSpinlock(h_Spinlock); +} + +uint32_t +XX_Sleep(uint32_t msecs) +{ + + XX_UDelay(1000 * msecs); + return (0); +} + +void +XX_UDelay(uint32_t usecs) +{ + DELAY(usecs); +} + +t_Error +XX_IpcRegisterMsgHandler(char addr[XX_IPC_MAX_ADDR_NAME_LENGTH], + t_IpcMsgHandler *f_MsgHandler, t_Handle h_Module, uint32_t replyLength) +{ + + /* + * This function returns fake E_OK status and does nothing + * as NetCommSW IPC is not used by FreeBSD drivers. + */ + return (E_OK); +} + +t_Error +XX_IpcUnregisterMsgHandler(char addr[XX_IPC_MAX_ADDR_NAME_LENGTH]) +{ + /* + * This function returns fake E_OK status and does nothing + * as NetCommSW IPC is not used by FreeBSD drivers. + */ + return (E_OK); +} + + +t_Error +XX_IpcSendMessage(t_Handle h_Session, + uint8_t *p_Msg, uint32_t msgLength, uint8_t *p_Reply, + uint32_t *p_ReplyLength, t_IpcMsgCompletion *f_Completion, t_Handle h_Arg) +{ + + /* Should not be called */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); + return (E_OK); +} + +t_Handle +XX_IpcInitSession(char destAddr[XX_IPC_MAX_ADDR_NAME_LENGTH], + char srcAddr[XX_IPC_MAX_ADDR_NAME_LENGTH]) +{ + + /* Should not be called */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); + return (NULL); +} + +t_Error +XX_IpcFreeSession(t_Handle h_Session) +{ + + /* Should not be called */ + printf("NetCommSW: Unimplemented function %s() called!\n", __func__); + return (E_OK); +} + +physAddress_t +XX_VirtToPhys(void *addr) +{ + vm_paddr_t paddr; + int cpu; + + cpu = PCPU_GET(cpuid); + + /* Handle NULL address */ + if (addr == NULL) + return (-1); + + /* + * No CCSR check needed on ARM64 — registers are mapped via + * bus_space, not through a global ccsrbar mapping. + */ + + /* Handle BMAN mappings */ + if (((vm_offset_t)addr >= XX_PInfo.portal_ce_va[BM_PORTAL]) && + ((vm_offset_t)addr < XX_PInfo.portal_ce_va[BM_PORTAL] + + XX_PInfo.portal_ce_size[BM_PORTAL][cpu])) + return (XX_PInfo.portal_ce_pa[BM_PORTAL][cpu] + + (vm_offset_t)addr - XX_PInfo.portal_ce_va[BM_PORTAL]); + + if (((vm_offset_t)addr >= XX_PInfo.portal_ci_va[BM_PORTAL]) && + ((vm_offset_t)addr < XX_PInfo.portal_ci_va[BM_PORTAL] + + XX_PInfo.portal_ci_size[BM_PORTAL][cpu])) + return (XX_PInfo.portal_ci_pa[BM_PORTAL][cpu] + + (vm_offset_t)addr - XX_PInfo.portal_ci_va[BM_PORTAL]); + + /* Handle QMAN mappings */ + if (((vm_offset_t)addr >= XX_PInfo.portal_ce_va[QM_PORTAL]) && + ((vm_offset_t)addr < XX_PInfo.portal_ce_va[QM_PORTAL] + + XX_PInfo.portal_ce_size[QM_PORTAL][cpu])) + return (XX_PInfo.portal_ce_pa[QM_PORTAL][cpu] + + (vm_offset_t)addr - XX_PInfo.portal_ce_va[QM_PORTAL]); + + if (((vm_offset_t)addr >= XX_PInfo.portal_ci_va[QM_PORTAL]) && + ((vm_offset_t)addr < XX_PInfo.portal_ci_va[QM_PORTAL] + + XX_PInfo.portal_ci_size[QM_PORTAL][cpu])) + return (XX_PInfo.portal_ci_pa[QM_PORTAL][cpu] + + (vm_offset_t)addr - XX_PInfo.portal_ci_va[QM_PORTAL]); + + /* ARM64 always has DMAP */ + if ((vm_offset_t)addr >= DMAP_MIN_ADDRESS && + (vm_offset_t)addr < DMAP_MAX_ADDRESS) + return (DMAP_TO_PHYS((vm_offset_t)addr)); + + paddr = pmap_kextract((vm_offset_t)addr); + + if (paddr == 0) + printf("NetCommSW: " + "Unable to translate virtual address %p!\n", addr); + + return (paddr); +} + +void * +XX_PhysToVirt(physAddress_t addr) +{ + int cpu; + + cpu = PCPU_GET(cpuid); + + /* + * No CCSR check needed on ARM64 — see XX_VirtToPhys comment. + */ + + /* Handle BMAN mappings */ + if ((addr >= XX_PInfo.portal_ce_pa[BM_PORTAL][cpu]) && + (addr < XX_PInfo.portal_ce_pa[BM_PORTAL][cpu] + + XX_PInfo.portal_ce_size[BM_PORTAL][cpu])) + return ((void *)(XX_PInfo.portal_ce_va[BM_PORTAL] + + (vm_offset_t)(addr - XX_PInfo.portal_ce_pa[BM_PORTAL][cpu]))); + + if ((addr >= XX_PInfo.portal_ci_pa[BM_PORTAL][cpu]) && + (addr < XX_PInfo.portal_ci_pa[BM_PORTAL][cpu] + + XX_PInfo.portal_ci_size[BM_PORTAL][cpu])) + return ((void *)(XX_PInfo.portal_ci_va[BM_PORTAL] + + (vm_offset_t)(addr - XX_PInfo.portal_ci_pa[BM_PORTAL][cpu]))); + + /* Handle QMAN mappings */ + if ((addr >= XX_PInfo.portal_ce_pa[QM_PORTAL][cpu]) && + (addr < XX_PInfo.portal_ce_pa[QM_PORTAL][cpu] + + XX_PInfo.portal_ce_size[QM_PORTAL][cpu])) + return ((void *)(XX_PInfo.portal_ce_va[QM_PORTAL] + + (vm_offset_t)(addr - XX_PInfo.portal_ce_pa[QM_PORTAL][cpu]))); + + if ((addr >= XX_PInfo.portal_ci_pa[QM_PORTAL][cpu]) && + (addr < XX_PInfo.portal_ci_pa[QM_PORTAL][cpu] + + XX_PInfo.portal_ci_size[QM_PORTAL][cpu])) + return ((void *)(XX_PInfo.portal_ci_va[QM_PORTAL] + + (vm_offset_t)(addr - XX_PInfo.portal_ci_pa[QM_PORTAL][cpu]))); + + /* + * Reject PA outside DRAM range (catches stale FD addresses from + * a previous boot and PA=0 which maps below dmap_phys_base). + */ + if (__predict_false(!PHYS_IN_DMAP_RANGE(addr))) + return (NULL); + return ((void *)(uintptr_t)PHYS_TO_DMAP(addr)); +} + +void +XX_PortalSetInfo(device_t dev) +{ + struct dpaa_portals_softc *sc; + const char *dev_name; + int i, type; + + dev_name = device_get_name(dev); + + if (strcmp(dev_name, "bman-portals") == 0) + type = BM_PORTAL; + else if (strcmp(dev_name, "qman-portals") == 0) + type = QM_PORTAL; + else + return; + + sc = device_get_softc(dev); + + for (i = 0; i < MAXCPU && sc->sc_dp[i].dp_ce_pa != 0; i++) { + XX_PInfo.portal_ce_pa[type][i] = sc->sc_dp[i].dp_ce_pa; + XX_PInfo.portal_ci_pa[type][i] = sc->sc_dp[i].dp_ci_pa; + XX_PInfo.portal_ce_size[type][i] = sc->sc_dp[i].dp_ce_size; + XX_PInfo.portal_ci_size[type][i] = sc->sc_dp[i].dp_ci_size; + XX_PInfo.portal_intr[type][i] = sc->sc_dp[i].dp_intr_num; + } + + XX_PInfo.portal_ce_va[type] = rman_get_bushandle(sc->sc_rres[0]); + XX_PInfo.portal_ci_va[type] = rman_get_bushandle(sc->sc_rres[1]); +} + +/* + * Stub for dead FManv3 clock-change code in fm_ncsw.c (FM_ChangeClock). + * fm_clk_down() is referenced but defined under #ifdef TODO_SOC_SUSPEND + * in fm_port.c — never compiled. g_MemacRegs is provided by memac.c. + */ +void fm_clk_down(void); +void +fm_clk_down(void) +{ + printf("NetCommSW: fm_clk_down() stub called — not implemented\n"); +} diff --git a/sys/dev/dpaa/bman.c b/sys/dev/dpaa/bman.c index 0dbcecb5d7a0..3d098a4e3d51 100644 --- a/sys/dev/dpaa/bman.c +++ b/sys/dev/dpaa/bman.c @@ -36,12 +36,25 @@ #include #include #include +#include +#include +#include +#ifdef __powerpc__ #include +#endif + +#ifdef __aarch64__ +#include +#include +#endif #include "bman.h" +#define FBPR_ENTRY_SIZE 64 /* matches NCSW bm.h */ + static struct bman_softc *bman_sc; +static t_Handle bman_pool_by_bpid[BM_MAX_NUM_OF_POOLS]; extern t_Handle bman_portal_setup(struct bman_softc *bsc); @@ -74,6 +87,7 @@ bman_exception(t_Handle h_App, e_BmExceptions exception) device_printf(sc->sc_dev, "BMAN Exception: %s.\n", message); } + int bman_attach(device_t dev) { @@ -114,12 +128,47 @@ bman_attach(device_t dev) bp.partBpidBase = 0; bp.partNumOfPools = BM_MAX_NUM_OF_POOLS; +#ifdef __aarch64__ + /* + * Parse DT memory-region phandle for FBPR, matching Linux. + * DT: memory-region = <&bman_fbpr>; + * bman-fbpr: size=0x1000000 (16MB) + */ + { + extern void *qbman_alloc_reserved_mem(device_t, phandle_t, + uint32_t *, const char *); + phandle_t node, mem_node; + pcell_t mem_handle; + + node = ofw_bus_get_node(dev); + if (node > 0 && OF_getencprop(node, "memory-region", + (void *)&mem_handle, sizeof(mem_handle)) == + sizeof(mem_handle)) { + mem_node = OF_node_from_xref(mem_handle); + if (mem_node > 0) { + bp.p_FbprBase = + qbman_alloc_reserved_mem(dev, + mem_node, &bp.fbprSize, "FBPR"); + if (bp.p_FbprBase != NULL) + bp.totalNumOfBuffers = + bp.fbprSize / FBPR_ENTRY_SIZE * 8; + } + } else { + device_printf(dev, + "no memory-region property, using defaults\n"); + } + } +#endif + + device_printf(dev, "CCSR va=%#lx totalBufs=%u\n", + (unsigned long)bp.baseAddress, bp.totalNumOfBuffers); + sc->sc_bh = BM_Config(&bp); if (sc->sc_bh == NULL) goto err; - /* Warn if there is less than 5% free FPBR's in pool */ - error = BM_ConfigFbprThreshold(sc->sc_bh, (BMAN_MAX_BUFFERS / 8) / 20); + /* Warn if there is less than 5% free FBPR's in pool */ + error = BM_ConfigFbprThreshold(sc->sc_bh, (bp.totalNumOfBuffers / 8) / 20); if (error != E_OK) goto err; @@ -134,6 +183,18 @@ bman_attach(device_t dev) device_printf(dev, "Hardware version: %d.%d.\n", rev.majorRev, rev.minorRev); +#ifdef __aarch64__ + /* + * Initialize BMan portals for all CPUs from boot CPU. + * ARM64: pmap_mapdev creates global kernel VA, no sched_bind needed. + */ + for (int cpu = 0; cpu < mp_ncpus; cpu++) { + if (bman_portal_init_cpu(sc, cpu) == NULL) + device_printf(dev, + "could not setup BMan portal on CPU %d\n", cpu); + } +#endif + return (0); err: @@ -255,6 +316,7 @@ bman_pool_create(uint8_t *bpid, uint16_t bufferSize, uint16_t maxBuffers, *bpid = BM_POOL_GetId(pool); sc->sc_bpool_cpu[*bpid] = PCPU_GET(cpuid); + bman_pool_by_bpid[*bpid] = pool; sched_unpin(); @@ -275,6 +337,7 @@ bman_pool_destroy(t_Handle pool) struct bman_softc *sc; sc = bman_sc; + bman_pool_by_bpid[BM_POOL_GetId(pool)] = NULL; thread_lock(curthread); sched_bind(curthread, sc->sc_bpool_cpu[BM_POOL_GetId(pool)]); thread_unlock(curthread); @@ -363,3 +426,12 @@ bman_count(t_Handle pool) return (BM_POOL_GetCounter(pool, e_BM_POOL_COUNTERS_CONTENT)); } + +t_Handle +bman_pool_for_bpid(uint8_t bpid) +{ + + if (bpid >= BM_MAX_NUM_OF_POOLS) + return (NULL); + return (bman_pool_by_bpid[bpid]); +} diff --git a/sys/dev/dpaa/bman.h b/sys/dev/dpaa/bman.h index 01c09489890c..36ecd2590c44 100644 --- a/sys/dev/dpaa/bman.h +++ b/sys/dev/dpaa/bman.h @@ -35,8 +35,8 @@ * BMAN Configuration */ -/* Maximum number of buffers in all BMAN pools */ -#define BMAN_MAX_BUFFERS 4096 +/* Maximum number of buffers in all BMAN pools (FBPR ring entries) */ +#define BMAN_MAX_BUFFERS 65536 /* * Portal definitions @@ -189,6 +189,7 @@ int bman_put_buffer(t_Handle pool, void *buffer); * @returns Number of free buffers in pool. */ uint32_t bman_count(t_Handle pool); +t_Handle bman_pool_for_bpid(uint8_t bpid); /* * Bus i/f @@ -199,4 +200,8 @@ int bman_suspend(device_t dev); int bman_resume(device_t dev); int bman_shutdown(device_t dev); +#ifdef __aarch64__ +t_Handle bman_portal_init_cpu(struct bman_softc *, int); +#endif + #endif /* BMAN_H */ diff --git a/sys/dev/dpaa/bman_fdt.c b/sys/dev/dpaa/bman_fdt.c index 2ff695bf7d89..44535e18f1fa 100644 --- a/sys/dev/dpaa/bman_fdt.c +++ b/sys/dev/dpaa/bman_fdt.c @@ -34,6 +34,8 @@ #include #include +#include +#include #include #include @@ -103,7 +105,10 @@ static driver_t bm_portals_driver = { }; EARLY_DRIVER_MODULE(bman_portals, ofwbus, bm_portals_driver, 0, 0, - BUS_PASS_BUS); + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LAST); +/* On ARM64, portal bus nodes are children of soc (simplebus), not ofwbus */ +EARLY_DRIVER_MODULE(bman_portals, simplebus, bm_portals_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LAST); static void get_addr_props(phandle_t node, uint32_t *addrp, uint32_t *sizep) @@ -147,7 +152,7 @@ bman_portal_find_cpu(int cpu) return (node); for (node = OF_child(node); node != 0; node = OF_peer(node)) { - if (OF_getprop(node, "reg", ®, sizeof(reg)) <= 0) + if (OF_getencprop(node, "reg", ®, sizeof(reg)) <= 0) continue; if (reg == cpu) return (node); @@ -184,14 +189,16 @@ bman_portals_fdt_attach(device_t dev) continue; } /* Checkout related cpu */ - if (OF_getprop(child, "cpu-handle", (void *)&cpu, + if (OF_getencprop(child, "cpu-handle", (void *)&cpu, sizeof(cpu)) <= 0) { - cpu = bman_portal_find_cpu(cpus); - if (cpu <= 0) + /* No cpu-handle (ARM64): find CPU node directly */ + cpu_node = bman_portal_find_cpu(cpus); + if (cpu_node <= 0) continue; + } else { + /* cpu-handle is an ihandle (PowerPC) */ + cpu_node = OF_instance_to_package(cpu); } - /* Acquire cpu number */ - cpu_node = OF_instance_to_package(cpu); if (OF_getencprop(cpu_node, "reg", &cpu_num, sizeof(cpu_num)) <= 0) { device_printf(dev, "Could not retrieve CPU number.\n"); return (ENXIO); diff --git a/sys/dev/dpaa/bman_portals.c b/sys/dev/dpaa/bman_portals.c index 89bcbf64ce5d..47f1d13f04c3 100644 --- a/sys/dev/dpaa/bman_portals.c +++ b/sys/dev/dpaa/bman_portals.c @@ -38,13 +38,16 @@ #include #include +#include +#include +#ifdef __powerpc__ #include +#include +#endif #include #include -#include - #include "bman.h" #include "portals.h" @@ -55,15 +58,16 @@ struct dpaa_portals_softc *bp_sc; int bman_portals_attach(device_t dev) { - struct dpaa_portals_softc *sc; + bp_sc = device_get_softc(dev); - sc = bp_sc = device_get_softc(dev); - - /* Map bman portal to physical address space */ +#ifdef __powerpc__ + struct dpaa_portals_softc *sc = bp_sc; + /* Map bman portal to physical address space (PowerPC LAW) */ if (law_enable(OCP85XX_TGTIF_BMAN, sc->sc_dp_pa, sc->sc_dp_size)) { bman_portals_detach(dev); return (ENXIO); } +#endif /* Set portal properties for XX_VirtToPhys() */ XX_PortalSetInfo(dev); @@ -146,8 +150,8 @@ bman_portal_setup(struct bman_softc *bsc) dpaa_portal_map_registers(sc); /* Configure and initialize portal */ - bpp.ceBaseAddress = rman_get_bushandle(sc->sc_rres[0]); - bpp.ciBaseAddress = rman_get_bushandle(sc->sc_rres[1]); + bpp.ceBaseAddress = sc->sc_dp[cpu].dp_ce_va; + bpp.ciBaseAddress = sc->sc_dp[cpu].dp_ci_va; bpp.h_Bm = bsc->sc_bh; bpp.swPortalId = cpu; bpp.irq = (uintptr_t)sc->sc_dp[cpu].dp_ires; @@ -174,3 +178,54 @@ bman_portal_setup(struct bman_softc *bsc) return (NULL); } + +#ifdef __aarch64__ +/* + * Initialize a BMan portal for a specific CPU from any CPU context. + * Unlike bman_portal_setup() which uses PCPU_GET(cpuid), this takes + * an explicit CPU parameter. On ARM64, pmap_mapdev creates global + * kernel VA, so no sched_bind/migration is needed. + * + * BMan has no SDEST configuration (confirmed from Linux reference). + */ +t_Handle +bman_portal_init_cpu(struct bman_softc *bsc, int cpu) +{ + struct dpaa_portals_softc *sc; + t_BmPortalParam bpp; + t_Handle portal; + + if (bp_sc == NULL) + return (NULL); + sc = bp_sc; + + /* Skip if already initialized */ + if (sc->sc_dp[cpu].dp_ph != NULL) + return (sc->sc_dp[cpu].dp_ph); + + /* Map CE/CI registers (global kernel VA, accessible from any CPU) */ + dpaa_portal_map_registers_cpu(sc, cpu); + + if (sc->sc_dp[cpu].dp_ce_va == 0 || sc->sc_dp[cpu].dp_ci_va == 0) + return (NULL); + + memset(&bpp, 0, sizeof(bpp)); + bpp.ceBaseAddress = sc->sc_dp[cpu].dp_ce_va; + bpp.ciBaseAddress = sc->sc_dp[cpu].dp_ci_va; + bpp.h_Bm = bsc->sc_bh; + bpp.swPortalId = cpu; + bpp.irq = (uintptr_t)sc->sc_dp[cpu].dp_ires; + + portal = BM_PORTAL_Config(&bpp); + if (portal == NULL) + return (NULL); + + if (BM_PORTAL_Init(portal) != E_OK) { + BM_PORTAL_Free(portal); + return (NULL); + } + + sc->sc_dp[cpu].dp_ph = portal; + return (portal); +} +#endif diff --git a/sys/dev/dpaa/dpaa_oh.c b/sys/dev/dpaa/dpaa_oh.c new file mode 100644 index 000000000000..44cf9cbe439e --- /dev/null +++ b/sys/dev/dpaa/dpaa_oh.c @@ -0,0 +1,736 @@ +/*- + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * DPAA1 Offline / Header Manipulation port driver. + * + * OH ports are FMan port instances without a MAC. Software enqueues frames + * via QMan, FMan processes them (parse, classify, manipulate), and the + * results come back via QMan. They serve as "on-ramps" to inject + * software-originated traffic (WiFi, IPsec) into the FMan pipeline. + * + * Each OH port probes a "fsl,fman-v3-port-oh" device tree node (child of + * the fman bus) and creates: + * - An FMan offline-parsing port (FM_PORT) + * - A default egress FQR (frames out of OH port after processing) + * - An error FQR (frames that hit errors during processing) + * - An ingress FQR (software enqueues here → FMan OH port dequeues) + * + * Consumers (CDX, WiFi, IPsec) use the public API in dpaa_oh.h to: + * - Get the FM_PORT handle for PCD attachment + * - Enqueue frames for offline processing + * - Register callbacks for processed frames + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "opt_dpaa.h" + +#include +#include +#include +#include + +#include "fman.h" +#include "fman_chardev.h" +#include "qman.h" +#include "dpaa_oh.h" + +/* From if_dtsec.h — avoid pulling in full header's if_t/uma deps */ +void dtsec_rm_buf_free_external(uint8_t bpid, void *buf); +void dtsec_rm_pool_rx_refill_bpid(uint8_t bpid); + + +/** + * @group OH port private defines. + * @{ + */ + +/* Frame queue parameters — same WQ as dtsec RX/TX */ +#define DPAA_OH_FQR_DFLT_WQ 1 +#define DPAA_OH_FQR_ERR_WQ 1 +#define DPAA_OH_FQR_TX_WQ 1 + +/* Pool channel for egress FQs (frames back to software) */ +#define DPAA_OH_FQR_DFLT_CHANNEL e_QM_FQ_CHANNEL_POOL1 +#define DPAA_OH_FQR_ERR_CHANNEL e_QM_FQ_CHANNEL_POOL1 + +/* LIODN base (same as dtsec) */ +#define DPAA_OH_LIODN_BASE 0 + +/* + * FreeBSD's upstream DTS (qoriq-fman3-0.dtsi) uses hardware port IDs as + * cell-index for OH ports: port@82000 has cell-index 2, port@83000 has + * cell-index 3, etc. NCSW FM_PORT_Config expects a 0-based relative port + * ID and adds BASE_OH_PORTID internally via SwPortIdToHwPortId(). + * + * The NXP Linux SDK DTS uses 0-based cell-indices (0, 1, 2, ...) instead, + * so the Linux wrapper passes cell-index directly without adjustment. + * + * We subtract the first OH port cell-index to convert from FreeBSD DTS + * convention to the 0-based relative ID that NCSW expects. + */ +#define DPAA_OH_CELL_INDEX_BASE 2 + +struct dpaa_oh_softc { + device_t sc_dev; + int sc_port_id; /* cell-index from DT (2-7) */ + uint32_t sc_port_hw_id; /* reg from DT (0x82000, etc.) */ + uint32_t sc_qman_chan; /* QMan channel for this port */ + + /* FMan handles (from fman parent) */ + vm_offset_t sc_fm_base; + t_Handle sc_fmh; + t_Handle sc_pcdh; + t_Handle sc_netenvh; + + /* FMan OH port */ + t_Handle sc_portph; /* FM_PORT handle */ + uint32_t sc_data_offset; /* buffer data offset (prefix size) */ + + /* Frame queues */ + t_Handle sc_dflt_fqr; /* default egress FQR */ + uint32_t sc_dflt_fqid; + t_Handle sc_err_fqr; /* error FQR */ + uint32_t sc_err_fqid; + t_Handle sc_tx_fqr; /* ingress FQR (software → OH) */ +}; +/** @} */ + + +/** + * @group FQR callbacks. + * @{ + */ + +static void +dpaa_oh_exception_callback(t_Handle app, e_FmPortExceptions exception) +{ + struct dpaa_oh_softc *sc; + + sc = app; + device_printf(sc->sc_dev, "FMan port exception %d\n", exception); +} + +static e_RxStoreResponse +dpaa_oh_dflt_cb(t_Handle app, t_Handle fqr, t_Handle portal, + uint32_t fqid_off, t_DpaaFD *frame) +{ + struct dpaa_oh_softc *sc; + void *buf; + + sc = app; + + /* + * Default FQ callback — frames that PCD didn't classify. + * For now, just drop and return the buffer. + * Consumers can override this via dpaa_oh_register_cb(). + */ + buf = DPAA_FD_GET_ADDR(frame); + if (buf != NULL) { + /* + * We don't own a buffer pool — the buffer was allocated + * by whoever enqueued the frame. We can't return it to + * BMan without knowing which pool it came from. The + * consumer's callback is responsible for buffer lifecycle. + * If no consumer is registered, we leak. This is a + * configuration error — OH ports should always have a + * consumer or PCD attached. + */ + if (bootverbose) + device_printf(sc->sc_dev, + "default FQ: unhandled frame (no consumer)\n"); + } + + return (e_RX_STORE_RESPONSE_CONTINUE); +} + +static e_RxStoreResponse +dpaa_oh_err_cb(t_Handle app, t_Handle fqr, t_Handle portal, + uint32_t fqid_off, t_DpaaFD *frame) +{ + struct dpaa_oh_softc *sc; + void *buf; + + sc = app; + device_printf(sc->sc_dev, "error frame: status 0x%08x bpid=%u\n", + DPAA_FD_GET_STATUS(frame), frame->bpid); + + buf = DPAA_FD_GET_ADDR(frame); + if (buf != NULL) { + dtsec_rm_buf_free_external(frame->bpid, buf); + dtsec_rm_pool_rx_refill_bpid(frame->bpid); + } + + return (e_RX_STORE_RESPONSE_CONTINUE); +} +/** @} */ + + +/** + * @group Driver probe/attach/detach. + * @{ + */ + +static int +dpaa_oh_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_is_compatible(dev, "fsl,fman-v3-port-oh")) + return (ENXIO); + + device_set_desc(dev, "Freescale FMan Offline Parsing Port"); + + return (BUS_PROBE_DEFAULT); +} + +static int +dpaa_oh_attach(device_t dev) +{ + struct dpaa_oh_softc *sc; + device_t parent; + t_FmPortParams params; + t_FmPortNonRxParams *oh_params; + t_FmBufferPrefixContent prefix; + t_Error error; + phandle_t node; + pcell_t cell_index; + pcell_t port_reg; + t_Handle fqr; + + sc = device_get_softc(dev); + sc->sc_dev = dev; + parent = device_get_parent(dev); + node = ofw_bus_get_node(dev); + + /* Read cell-index — the hardware port ID for OH ports (2-7) */ + if (OF_getencprop(node, "cell-index", &cell_index, + sizeof(cell_index)) <= 0) { + device_printf(dev, "missing cell-index property\n"); + return (ENXIO); + } + sc->sc_port_id = cell_index; + + /* Read reg — hardware register offset within FMan */ + if (OF_getencprop(node, "reg", &port_reg, sizeof(port_reg)) <= 0) { + device_printf(dev, "missing reg property\n"); + return (ENXIO); + } + sc->sc_port_hw_id = port_reg; + + /* Get FMan handles from parent */ + if (fman_get_handle(parent, &sc->sc_fmh) != 0 || + sc->sc_fmh == NULL) { + device_printf(dev, "couldn't get FMan handle\n"); + return (ENXIO); + } + + if (fman_get_bushandle(parent, &sc->sc_fm_base) != 0) { + device_printf(dev, "couldn't get FMan base address\n"); + return (ENXIO); + } + + fman_get_pcd_handle(parent, &sc->sc_pcdh); + fman_get_netenv_handle(parent, &sc->sc_netenvh); + + /* Get QMan channel for this OH port */ + sc->sc_qman_chan = fman_qman_channel_id(parent, sc->sc_port_id); + if (sc->sc_qman_chan == 0) { + device_printf(dev, + "couldn't get QMan channel for port %d\n", + sc->sc_port_id); + return (ENXIO); + } + + /* + * Create frame queues. + */ + + /* Default egress FQR — processed frames come here */ + fqr = qman_fqr_create(1, DPAA_OH_FQR_DFLT_CHANNEL, + DPAA_OH_FQR_DFLT_WQ, false, 0, false, false, true, false, + 0, 0, 0); + if (fqr == NULL) { + device_printf(dev, "couldn't create default FQR\n"); + return (EIO); + } + sc->sc_dflt_fqr = fqr; + sc->sc_dflt_fqid = qman_fqr_get_base_fqid(fqr); + + error = qman_fqr_register_cb(fqr, dpaa_oh_dflt_cb, sc); + if (error != E_OK) { + device_printf(dev, "couldn't register default FQ callback\n"); + goto fail_dflt; + } + + /* Error FQR */ + fqr = qman_fqr_create(1, DPAA_OH_FQR_ERR_CHANNEL, + DPAA_OH_FQR_ERR_WQ, false, 0, false, false, true, false, + 0, 0, 0); + if (fqr == NULL) { + device_printf(dev, "couldn't create error FQR\n"); + goto fail_dflt; + } + sc->sc_err_fqr = fqr; + sc->sc_err_fqid = qman_fqr_get_base_fqid(fqr); + + error = qman_fqr_register_cb(fqr, dpaa_oh_err_cb, sc); + if (error != E_OK) { + device_printf(dev, "couldn't register error FQ callback\n"); + goto fail_err; + } + + /* Ingress FQR — software enqueues here, FMan OH port dequeues */ + fqr = qman_fqr_create(1, sc->sc_qman_chan, + DPAA_OH_FQR_TX_WQ, false, 0, false, false, true, false, + 0, 0, 0); + if (fqr == NULL) { + device_printf(dev, "couldn't create ingress FQR\n"); + goto fail_err; + } + sc->sc_tx_fqr = fqr; + + /* + * Configure FMan OH port. + * + * OH ports use nonRxParams (like TX ports) because they receive + * frames from QMan (not from a MAC). But they also need dfltFqid + * (like RX ports) for the egress path. + */ + memset(¶ms, 0, sizeof(params)); + params.baseAddr = sc->sc_fm_base + sc->sc_port_hw_id; + params.h_Fm = sc->sc_fmh; + params.portType = e_FM_PORT_TYPE_OH_OFFLINE_PARSING; + params.portId = sc->sc_port_id - DPAA_OH_CELL_INDEX_BASE; + params.independentModeEnable = false; + params.liodnBase = DPAA_OH_LIODN_BASE; + params.f_Exception = dpaa_oh_exception_callback; + params.h_App = sc; + + oh_params = ¶ms.specificParams.nonRxParams; + oh_params->errFqid = sc->sc_err_fqid; + oh_params->dfltFqid = sc->sc_dflt_fqid; + oh_params->qmChannel = sc->sc_qman_chan; +#ifdef FM_OP_PARTITION_ERRATA_FMANx8 + oh_params->opLiodnOffset = 0; +#endif + + sc->sc_portph = FM_PORT_Config(¶ms); + if (sc->sc_portph == NULL) { + device_printf(dev, "couldn't configure FM OH port\n"); + goto fail_tx; + } + + /* + * Configure buffer prefix: parse result + hash result. + * This enables PCD (parser writes results into prefix) and + * allows consumers to read parse results for classification. + * + * manipExtraSpace = 96 matches the Linux SDK DTS + * buffer-layout = <0x60 0x40> for OH port extended-args. + */ + memset(&prefix, 0, sizeof(prefix)); + prefix.privDataSize = 16; + prefix.passPrsResult = TRUE; + prefix.passHashResult = TRUE; + prefix.dataAlign = 64; + prefix.manipExtraSpace = 96; + + error = FM_PORT_ConfigBufferPrefixContent(sc->sc_portph, &prefix); + if (error != E_OK) { + device_printf(dev, "couldn't configure buffer prefix\n"); + goto fail_port; + } + + error = FM_PORT_Init(sc->sc_portph); + if (error != E_OK) { + device_printf(dev, "couldn't initialize FM OH port: %d\n", + error); + goto fail_port; + } + + sc->sc_data_offset = FM_PORT_GetBufferDataOffset(sc->sc_portph); + + /* Register OH port with FMD chardev for userspace PCD control */ + { + struct fmcd_softc *fmcd; + + fmcd = fman_get_fmcd(device_get_parent(dev)); + if (fmcd != NULL) + fmcd_register_oh_port(fmcd, + sc->sc_port_id - DPAA_OH_CELL_INDEX_BASE, + sc->sc_portph); + } + + device_printf(dev, + "OH port %d: QMan chan 0x%x, dflt FQID %u, err FQID %u, " + "data offset %u\n", + sc->sc_port_id, sc->sc_qman_chan, sc->sc_dflt_fqid, + sc->sc_err_fqid, sc->sc_data_offset); + + return (0); + +fail_port: + FM_PORT_Free(sc->sc_portph); + sc->sc_portph = NULL; +fail_tx: + qman_fqr_free(sc->sc_tx_fqr); + sc->sc_tx_fqr = NULL; +fail_err: + qman_fqr_free(sc->sc_err_fqr); + sc->sc_err_fqr = NULL; +fail_dflt: + qman_fqr_free(sc->sc_dflt_fqr); + sc->sc_dflt_fqr = NULL; + return (ENXIO); +} + +static int +dpaa_oh_detach(device_t dev) +{ + struct dpaa_oh_softc *sc; + + sc = device_get_softc(dev); + + if (sc->sc_portph != NULL) { + FM_PORT_Free(sc->sc_portph); + sc->sc_portph = NULL; + } + + if (sc->sc_tx_fqr != NULL) + qman_fqr_free(sc->sc_tx_fqr); + if (sc->sc_err_fqr != NULL) + qman_fqr_free(sc->sc_err_fqr); + if (sc->sc_dflt_fqr != NULL) + qman_fqr_free(sc->sc_dflt_fqr); + + return (0); +} +/** @} */ + + +/** + * @group Distribution FQ callback registry. + * + * BPID-keyed table for OH port consumers to register distribution FQ + * handlers. CDX calls dpaa_oh_lookup_dist_cb() in its distribution + * FQ callback to dispatch frames to the appropriate consumer. + * @{ + */ +#define DPAA_OH_MAX_DIST_CBS 4 + +static struct { + uint8_t bpid; + bool active; + dpaa_oh_dist_cb_t fn; + t_Handle app; +} dpaa_oh_dist_cbs[DPAA_OH_MAX_DIST_CBS]; + +int +dpaa_oh_register_dist_cb(uint8_t bpid, dpaa_oh_dist_cb_t fn, t_Handle app) +{ + int i; + + for (i = 0; i < DPAA_OH_MAX_DIST_CBS; i++) { + if (!dpaa_oh_dist_cbs[i].active) { + dpaa_oh_dist_cbs[i].bpid = bpid; + dpaa_oh_dist_cbs[i].fn = fn; + dpaa_oh_dist_cbs[i].app = app; + atomic_thread_fence_rel(); + dpaa_oh_dist_cbs[i].active = true; + printf("dpaa_oh: dist cb registered for bpid %u\n", + bpid); + return (0); + } + } + return (ENOSPC); +} + +void +dpaa_oh_unregister_dist_cb(uint8_t bpid) +{ + int i; + + for (i = 0; i < DPAA_OH_MAX_DIST_CBS; i++) { + if (dpaa_oh_dist_cbs[i].active && + dpaa_oh_dist_cbs[i].bpid == bpid) { + dpaa_oh_dist_cbs[i].active = false; + atomic_thread_fence_rel(); + printf("dpaa_oh: dist cb unregistered for bpid %u\n", + bpid); + return; + } + } +} + +dpaa_oh_dist_cb_t +dpaa_oh_lookup_dist_cb(uint8_t bpid, t_Handle *app) +{ + int i; + + for (i = 0; i < DPAA_OH_MAX_DIST_CBS; i++) { + if (dpaa_oh_dist_cbs[i].active && + dpaa_oh_dist_cbs[i].bpid == bpid) { + *app = dpaa_oh_dist_cbs[i].app; + return (dpaa_oh_dist_cbs[i].fn); + } + } + return (NULL); +} + +/* + * Fallback callback for distribution FQ frames that don't match + * any BPID-specific registration. Used by dpaa_wifi to handle + * CDX→WiFi frames carrying dtsec BPIDs on OH port dist FQs. + */ +static dpaa_oh_dist_cb_t dpaa_oh_dist_fallback_fn; +static t_Handle dpaa_oh_dist_fallback_app; + +int +dpaa_oh_register_dist_fallback(dpaa_oh_dist_cb_t fn, t_Handle app) +{ + + dpaa_oh_dist_fallback_fn = fn; + dpaa_oh_dist_fallback_app = app; + atomic_thread_fence_rel(); + printf("dpaa_oh: dist fallback callback registered\n"); + return (0); +} + +dpaa_oh_dist_cb_t +dpaa_oh_lookup_dist_fallback(t_Handle *app) +{ + dpaa_oh_dist_cb_t fn; + + fn = dpaa_oh_dist_fallback_fn; + if (fn != NULL) + *app = dpaa_oh_dist_fallback_app; + return (fn); +} +/** @} */ + + +/** + * @group Public API. + * @{ + */ + +device_t +dpaa_oh_find_port(int cell_index) +{ + devclass_t dc; + device_t *devlist; + int count, i; + device_t result; + + dc = devclass_find("dpaa_oh"); + if (dc == NULL) + return (NULL); + + if (devclass_get_devices(dc, &devlist, &count) != 0) + return (NULL); + + result = NULL; + for (i = 0; i < count; i++) { + struct dpaa_oh_softc *sc; + + sc = device_get_softc(devlist[i]); + if (sc->sc_port_id == cell_index) { + result = devlist[i]; + break; + } + } + + free(devlist, M_TEMP); + return (result); +} + +t_Handle +dpaa_oh_get_fm_port(device_t dev) +{ + struct dpaa_oh_softc *sc; + + sc = device_get_softc(dev); + return (sc->sc_portph); +} + +uint32_t +dpaa_oh_get_qman_channel(device_t dev) +{ + struct dpaa_oh_softc *sc; + + sc = device_get_softc(dev); + return (sc->sc_qman_chan); +} + +uint32_t +dpaa_oh_get_default_fqid(device_t dev) +{ + struct dpaa_oh_softc *sc; + + sc = device_get_softc(dev); + return (sc->sc_dflt_fqid); +} + +uint32_t +dpaa_oh_get_data_offset(device_t dev) +{ + struct dpaa_oh_softc *sc; + + sc = device_get_softc(dev); + return (sc->sc_data_offset); +} + +int +dpaa_oh_enqueue(device_t dev, t_DpaaFD *fd) +{ + struct dpaa_oh_softc *sc; + t_Error error; + + sc = device_get_softc(dev); + if (sc->sc_tx_fqr == NULL) + return (ENXIO); + + error = qman_fqr_enqueue(sc->sc_tx_fqr, 0, fd); + if (error == E_OK) + return (0); + if (GET_ERROR_TYPE(error) == E_BUSY) + return (EBUSY); + return (EIO); +} + +int +dpaa_oh_register_cb(device_t dev, t_QmReceivedFrameCallback *callback, + t_Handle app) +{ + struct dpaa_oh_softc *sc; + t_Error error; + + sc = device_get_softc(dev); + if (sc->sc_dflt_fqr == NULL) + return (ENXIO); + + error = qman_fqr_register_cb(sc->sc_dflt_fqr, callback, app); + if (error != E_OK) + return (EIO); + + return (0); +} +/** @} */ + + +/** + * @group Device driver boilerplate. + * @{ + */ + +static device_method_t dpaa_oh_methods[] = { + DEVMETHOD(device_probe, dpaa_oh_probe), + DEVMETHOD(device_attach, dpaa_oh_attach), + DEVMETHOD(device_detach, dpaa_oh_detach), + { 0, 0 } +}; + +static driver_t dpaa_oh_driver = { + "dpaa_oh", + dpaa_oh_methods, + sizeof(struct dpaa_oh_softc), +}; + +DRIVER_MODULE(dpaa_oh, fman, dpaa_oh_driver, 0, 0); +MODULE_VERSION(dpaa_oh, 1); + +/* + * Per-VAP distribution FQ registry. + * + * dpaa_wifi registers base FQID + count for each WiFi VAP. + * CDX queries dpaa_oh_get_vap_fwd_fqid() to get hash-distributed + * FQIDs for CDX→WiFi download frames. + */ +static struct { + uint32_t base_fqid; + uint32_t count; /* must be power of 2 */ + bool active; +} dpaa_oh_vap_fwd[DPAA_OH_MAX_VAPS]; + +int +dpaa_oh_register_vap_fwd_fqs(int vap_idx, uint32_t base_fqid, + uint32_t count) +{ + + if (vap_idx < 0 || vap_idx >= DPAA_OH_MAX_VAPS) + return (EINVAL); + if ((count & (count - 1)) != 0) + return (EINVAL); /* must be power of 2 */ + + dpaa_oh_vap_fwd[vap_idx].base_fqid = base_fqid; + dpaa_oh_vap_fwd[vap_idx].count = count; + atomic_thread_fence_rel(); + dpaa_oh_vap_fwd[vap_idx].active = true; + + printf("dpaa_oh: vap %d FWD FQs registered: base %u, count %u\n", + vap_idx, base_fqid, count); + return (0); +} + +void +dpaa_oh_unregister_vap_fwd_fqs(int vap_idx) +{ + + if (vap_idx >= 0 && vap_idx < DPAA_OH_MAX_VAPS) { + dpaa_oh_vap_fwd[vap_idx].active = false; + atomic_thread_fence_rel(); + } +} + +int +dpaa_oh_get_vap_fwd_fqid(int vap_idx, uint32_t *fqid, uint32_t hash) +{ + + if (vap_idx < 0 || vap_idx >= DPAA_OH_MAX_VAPS || + !dpaa_oh_vap_fwd[vap_idx].active) + return (-1); + + *fqid = dpaa_oh_vap_fwd[vap_idx].base_fqid + + (hash & (dpaa_oh_vap_fwd[vap_idx].count - 1)); + return (0); +} +/** @} */ diff --git a/sys/dev/dpaa/dpaa_oh.h b/sys/dev/dpaa/dpaa_oh.h new file mode 100644 index 000000000000..dc46dac1c7f2 --- /dev/null +++ b/sys/dev/dpaa/dpaa_oh.h @@ -0,0 +1,139 @@ +/*- + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef DPAA_OH_H_ +#define DPAA_OH_H_ + +#include +#include +#include +#include + +/** + * Find an OH port device by its cell-index (hardware port ID). + * Cell-index values are 2-7 for FManv3 OH ports. + * + * @param cell_index The cell-index from device tree. + * @return The device_t, or NULL if not found. + */ +device_t dpaa_oh_find_port(int cell_index); + +/** + * Get the FM_PORT handle for PCD attachment. + * Consumers (e.g., CDX) use this to call FM_PORT_SetPCD(). + */ +t_Handle dpaa_oh_get_fm_port(device_t dev); + +/** + * Get the QMan channel ID for this OH port. + * Consumers create FQs with this channel as destination to send + * frames to the OH port for processing. + */ +uint32_t dpaa_oh_get_qman_channel(device_t dev); + +/** + * Get the default (egress) FQID. + * Frames that pass through the OH port without PCD classification + * are enqueued to this FQID. + */ +uint32_t dpaa_oh_get_default_fqid(device_t dev); + +/** + * Get the buffer data offset (prefix size). + * When building frames to send to the OH port, data must start + * at this offset within the buffer to leave room for the prefix. + */ +uint32_t dpaa_oh_get_data_offset(device_t dev); + +/** + * Enqueue a frame descriptor to the OH port for processing. + * + * @param dev The OH port device. + * @param fd The frame descriptor to enqueue. + * @return 0 on success, EBUSY if the portal EQCR ring is full + * (caller should retry), EIO on other errors, ENXIO if + * the port is not initialized. + */ +int dpaa_oh_enqueue(device_t dev, t_DpaaFD *fd); + +/** + * Register a callback for frames exiting the OH port's default FQ. + * These are "miss" frames that PCD did not classify into a flow. + * + * @param dev The OH port device. + * @param callback The callback function (same signature as QMan RX cb). + * @param app Opaque argument passed to callback. + * @return 0 on success, errno on failure. + */ +int dpaa_oh_register_cb(device_t dev, + t_QmReceivedFrameCallback *callback, t_Handle app); + +/** + * Distribution FQ callback type. + * Same signature as t_QmReceivedFrameCallback. + */ +typedef e_RxStoreResponse (*dpaa_oh_dist_cb_t)(t_Handle app, + t_Handle fqr, t_Handle portal, uint32_t fqid_off, t_DpaaFD *fd); + +/** + * Register a distribution FQ callback keyed by BMan pool ID. + * + * When CDX takes over PCD, frames matching KeyGen distribution + * schemes on OH ports arrive at CDX distribution FQs. OH port + * consumers (WiFi, IPsec) register here so CDX can dispatch + * these frames instead of dropping them. + * + * @param bpid BMan pool ID that identifies the consumer's frames. + * @param fn Callback function for frame delivery. + * @param app Opaque argument passed to callback. + * @return 0 on success, ENOSPC if registry is full. + */ +int dpaa_oh_register_dist_cb(uint8_t bpid, + dpaa_oh_dist_cb_t fn, t_Handle app); +void dpaa_oh_unregister_dist_cb(uint8_t bpid); +dpaa_oh_dist_cb_t dpaa_oh_lookup_dist_cb(uint8_t bpid, t_Handle *app); + +int dpaa_oh_register_dist_fallback(dpaa_oh_dist_cb_t fn, + t_Handle app); +dpaa_oh_dist_cb_t dpaa_oh_lookup_dist_fallback(t_Handle *app); + +/* + * Per-VAP distribution FQ registry. + * + * dpaa_wifi creates per-VAP FQs for CDX→WiFi download distribution + * (matching Linux VWD wlan_fq_from_fman[64]). It registers the base + * FQID + count here. CDX queries this to get hash-distributed FQIDs. + */ +#define DPAA_OH_MAX_VAPS 4 +#define DPAA_OH_FWD_FQ_MAX 64 + +int dpaa_oh_register_vap_fwd_fqs(int vap_idx, + uint32_t base_fqid, uint32_t count); +void dpaa_oh_unregister_vap_fwd_fqs(int vap_idx); +int dpaa_oh_get_vap_fwd_fqid(int vap_idx, uint32_t *fqid, + uint32_t hash); + +#endif /* DPAA_OH_H_ */ diff --git a/sys/dev/dpaa/fman.c b/sys/dev/dpaa/fman.c index f14ca45e1fc1..42079c282820 100644 --- a/sys/dev/dpaa/fman.c +++ b/sys/dev/dpaa/fman.c @@ -38,19 +38,33 @@ #include #include +#include + +#include +#include #include "opt_platform.h" +#include "opt_dpaa.h" #include #include +#include #include +#include #include #include "fman.h" +#include "fman_chardev.h" +#include + +/* NCSW internal headers — needed for hot-adding HC to existing PCD */ +#include +#include static MALLOC_DEFINE(M_FMAN, "fman", "fman devices information"); + /** * @group FMan private defines. * @{ @@ -62,7 +76,11 @@ enum fman_irq_enum { enum fman_mu_ram_map { FMAN_MURAM_OFF = 0x0, - FMAN_MURAM_SIZE = 0x28000 +#ifdef __aarch64__ + FMAN_MURAM_SIZE = 0x60000 /* LS1046A FManv3: 384KB */ +#else + FMAN_MURAM_SIZE = 0x28000 /* P5020 FManv2: 160KB */ +#endif }; struct fman_config { @@ -81,11 +99,13 @@ struct fman_config { */ /** * Frame Manager firmware. - * We use the same firmware for both P3041 and P2041 devices. + * On ARM64 (LS1046A), U-Boot preloads FManv3 microcode into IRAM, + * so we skip embedding firmware. On PowerPC, use the P3041 image. */ +#ifndef __aarch64__ const uint32_t fman_firmware[] = FMAN_UC_IMG; const uint32_t fman_firmware_size = sizeof(fman_firmware); - +#endif int fman_activate_resource(device_t bus, device_t child, int type, int rid, struct resource *res) @@ -260,6 +280,110 @@ fman_fill_ranges(phandle_t node, struct simplebus_softc *sc) return (sc->nranges); } +static void +fman_pcd_exception_callback(t_Handle app_handle, + e_FmPcdExceptions exception) +{ + struct fman_softc *sc; + + sc = app_handle; + device_printf(sc->sc_base.dev, "PCD exception %d occurred.\n", + exception); +} + +static void +fman_pcd_id_exception_callback(t_Handle app_handle, + e_FmPcdExceptions exception, uint16_t index) +{ + struct fman_softc *sc; + + sc = app_handle; + device_printf(sc->sc_base.dev, + "PCD indexed exception %d, index %u.\n", exception, index); +} + +/* + * Host Command (HC) port parameters. + * + * The HC port uses OH port cell-index 0x2 (the first offline port), + * which is reserved in the Mono Gateway DTS with status="disabled" + * so dpaa_oh doesn't claim it. FMANv3 requires portId=0 for HC. + */ +#define FMAN_HC_OH_REG_OFFSET 0x82000 /* OH port 0x2 register offset */ +#define FMAN_HC_PORT_ID 0 /* MUST be 0 for FMANv3 */ +#define FMAN_HC_OH_CELL_INDEX 0x02 /* DT cell-index for QMan channel */ +#define FMAN_HC_LIODN_BASE 0 /* ARM64 SMMU handles LIODN */ +#define FMAN_HC_FQ_WQ 1 /* Work queue priority */ + +/* + * Byte-swap every uint32_t in a buffer. + * + * FMan is big-endian and reads HC frame data via DMA with no hardware + * swap (DMA_NO_SWAP). On PPC (big-endian), native uint32_t values + * are already in the correct byte order. On ARM64 (little-endian), + * every uint32_t must be byte-swapped so FMan reads correct values. + * + * The Linux NXP SDK does exactly this in QmEnqueueCB (before enqueue) + * and qm_tx_conf_dqrr_cb (after confirmation) in lnxwrp_fm_port.c. + */ +static __inline void +fman_hc_bswap_frame(void *buf, uint32_t len) +{ +#if _BYTE_ORDER == _LITTLE_ENDIAN + uint32_t *p = (uint32_t *)buf; + uint32_t i; + + for (i = 0; i < len / 4; i++) + p[i] = __builtin_bswap32(p[i]); +#endif +} + +/* + * QMan enqueue callback for the HC port. + * Called by NCSW HC code (hc.c EnQFrm) to send command frames + * through QMan to the FMan HC offline port. + */ +static t_Error +fman_hc_qm_enqueue(t_Handle h_arg, void *p_fd) +{ + struct fman_softc *sc; + t_DpaaFD *fd = (t_DpaaFD *)p_fd; + + sc = h_arg; + + /* Byte-swap HC frame data for FMan big-endian DMA */ + fman_hc_bswap_frame(DPAA_FD_GET_ADDR(fd), DPAA_FD_GET_LENGTH(fd)); + + return (qman_fqr_enqueue(sc->hc_tx_fqr, 0, fd)); +} + +/* + * QMan receive callback for HC confirmation frames. + * + * FMan sends HC command completions on the confirmation FQ. QMan + * delivers them here; we forward to FM_PCD_HcTxConf() which clears + * the "enqueued" flag in hc.c, unblocking the spinning EnQFrm(). + * + * The HC frame data was byte-swapped before enqueue (for FMan BE DMA). + * Swap it back to native byte order so FmHcTxConf can read fields + * like commandSequence in native format. + */ +static e_RxStoreResponse +fman_hc_conf_callback(t_Handle app, t_Handle qm_fqr, t_Handle qm_portal, + uint32_t fqid_offset, t_DpaaFD *frame) +{ + struct fman_softc *sc; + + sc = app; + + /* Byte-swap HC frame data back to native byte order */ + fman_hc_bswap_frame(DPAA_FD_GET_ADDR(frame), + DPAA_FD_GET_LENGTH(frame)); + + FM_PCD_HcTxConf(sc->pcd_handle, frame); + return (e_RX_STORE_RESPONSE_CONTINUE); +} + static t_Handle fman_init(struct fman_softc *sc, struct fman_config *cfg) { @@ -271,6 +395,8 @@ fman_init(struct fman_softc *sc, struct fman_config *cfg) uint16_t clock; uint32_t tmp, mod; + memset(&fm_params, 0, sizeof(fm_params)); + /* MURAM configuration */ muram_handle = FM_MURAM_ConfigAndInit(cfg->mem_base_addr + FMAN_MURAM_OFF, FMAN_MURAM_SIZE); @@ -281,6 +407,12 @@ fman_init(struct fman_softc *sc, struct fman_config *cfg) } sc->muram_handle = muram_handle; + /* + * MURAM stays as Device-nGnRnE (the default bus resource mapping), + * matching Linux devm_ioremap(). All MURAM accesses must use + * WRITE_UINT32/GET_UINT32/IOMemSet32 — never plain memset/memcpy. + */ + /* Fill in FM configuration */ fm_params.fmId = cfg->fm_id; /* XXX we support only one partition thus each fman has master id */ @@ -307,8 +439,22 @@ fman_init(struct fman_softc *sc, struct fman_config *cfg) fm_params.irq = cfg->irq_num; fm_params.errIrq = cfg->err_irq_num; +#ifdef __aarch64__ + /* U-Boot preloads FMan microcode into IRAM; skip firmware loading */ + fm_params.firmware.size = 0; + fm_params.firmware.p_Code = NULL; +#else fm_params.firmware.size = fman_firmware_size; fm_params.firmware.p_Code = (uint32_t*)fman_firmware; +#endif + +#if (DPAA_VERSION >= 11) + /* FManv3 Virtual Storage Profiles — master owns all 64 entries. + * VSP register block at FMan base + 0xdc000 (from fm_common.h FM_MM_SP). */ + fm_params.partVSPBase = 0; + fm_params.partNumOfVSPs = FM_VSP_MAX_NUM_OF_ENTRIES; + fm_params.vspBaseAddr = cfg->mem_base_addr + 0xdc000; +#endif fm_handle = FM_Config(&fm_params); if (fm_handle == NULL) { @@ -317,7 +463,12 @@ fman_init(struct fman_softc *sc, struct fman_config *cfg) goto err; } +#ifdef __aarch64__ + /* Don't reset FMan — would wipe U-Boot's pre-loaded microcode */ + FM_ConfigResetOnInit(fm_handle, FALSE); +#else FM_ConfigResetOnInit(fm_handle, TRUE); +#endif error = FM_Init(fm_handle); if (error != E_OK) { @@ -334,6 +485,100 @@ fman_init(struct fman_softc *sc, struct fman_config *cfg) device_printf(cfg->fman_device, "Hardware version: %d.%d.\n", revision_info.majorRev, revision_info.minorRev); + { + t_FmCtrlCodeRevisionInfo ucode_rev; + if (FM_GetFmanCtrlCodeRevision(fm_handle, &ucode_rev) == E_OK) + device_printf(cfg->fman_device, + "Microcode: package %u, version %u.%u%s\n", + ucode_rev.packageRev, + ucode_rev.majorRev, ucode_rev.minorRev, + ucode_rev.packageRev >= 209 ? " (CDX/ASK)" : + ucode_rev.packageRev == 106 ? " (IP offload)" : + " (unknown)"); + } + + /* + * Initialize PCD (Parse-Classify-Distribute) with Parser + + * KeyGen + CC for RSS/multi-queue RX distribution and + * classification support. + * + * CC is enabled here so CDX can use it for hash tables. + * Host Command (HC) is NOT enabled here because QMan portals + * aren't ready during early FMan attach. CDX calls + * fman_reinit_pcd_with_hc() later to hot-add HC to this PCD + * without rebuilding it — preserving existing dtsec schemes. + */ + { + t_FmPcdParams pcd_params; + t_FmPcdNetEnvParams netenv_params; + + sc->pcd_handle = NULL; + sc->netenv_handle = NULL; + sc->hc_tx_fqr = NULL; + sc->hc_err_fqr = NULL; + sc->hc_conf_fqr = NULL; + + memset(&pcd_params, 0, sizeof(pcd_params)); + pcd_params.h_Fm = fm_handle; + pcd_params.prsSupport = TRUE; + pcd_params.ccSupport = TRUE; + pcd_params.kgSupport = TRUE; + pcd_params.plcrSupport = TRUE; + pcd_params.numOfSchemes = FM_PCD_KG_NUM_OF_SCHEMES; + pcd_params.f_Exception = fman_pcd_exception_callback; + pcd_params.f_ExceptionId = fman_pcd_id_exception_callback; + pcd_params.h_App = sc; + + sc->pcd_handle = FM_PCD_Config(&pcd_params); + if (sc->pcd_handle == NULL) { + device_printf(cfg->fman_device, + "couldn't configure PCD\n"); + goto skip_pcd; + } + + FM_PCD_ConfigPlcrNumOfSharedProfiles(sc->pcd_handle, 16); + + error = FM_PCD_Init(sc->pcd_handle); + if (error != E_OK) { + device_printf(cfg->fman_device, + "couldn't init PCD: %d\n", error); + FM_PCD_Free(sc->pcd_handle); + sc->pcd_handle = NULL; + goto skip_pcd; + } + + error = FM_PCD_Enable(sc->pcd_handle); + if (error != E_OK) { + device_printf(cfg->fman_device, + "couldn't enable PCD: %d\n", error); + FM_PCD_Free(sc->pcd_handle); + sc->pcd_handle = NULL; + goto skip_pcd; + } + + /* + * Network environment: IPv4/IPv6 + TCP/UDP distinction. + * This tells PCD which protocol layers to recognize. + */ + memset(&netenv_params, 0, sizeof(netenv_params)); + netenv_params.numOfDistinctionUnits = 2; + netenv_params.units[0].hdrs[0].hdr = HEADER_TYPE_IPv4; + netenv_params.units[0].hdrs[1].hdr = HEADER_TYPE_IPv6; + netenv_params.units[1].hdrs[0].hdr = HEADER_TYPE_TCP; + netenv_params.units[1].hdrs[1].hdr = HEADER_TYPE_UDP; + + sc->netenv_handle = FM_PCD_NetEnvCharacteristicsSet( + sc->pcd_handle, &netenv_params); + if (sc->netenv_handle == NULL) + device_printf(cfg->fman_device, + "couldn't set NetEnv characteristics\n"); + + device_printf(cfg->fman_device, + "PCD initialized: Parser + KeyGen\n"); +skip_pcd: + ; + } + /* Initialize the simplebus part of things */ simplebus_init(sc->sc_base.dev, 0); @@ -352,6 +597,7 @@ fman_init(struct fman_softc *sc, struct fman_config *cfg) FM_Free(fm_handle); err: FM_MURAM_Free(muram_handle); + sc->muram_handle = NULL; return (NULL); } @@ -361,7 +607,7 @@ fman_exception_callback(t_Handle app_handle, e_FmExceptions exception) struct fman_softc *sc; sc = app_handle; - device_printf(sc->sc_base.dev, "FMan exception occurred.\n"); + device_printf(sc->sc_base.dev, "FMan exception %d\n", exception); } static void @@ -371,8 +617,11 @@ fman_error_callback(t_Handle app_handle, e_FmPortType port_type, struct fman_softc *sc; sc = app_handle; - device_printf(sc->sc_base.dev, "FMan error occurred.\n"); + device_printf(sc->sc_base.dev, + "FMan error: portType=%d portId=%u addr=0x%llx tnum=%u liodn=%u\n", + port_type, port_id, (unsigned long long)addr, tnum, liodn); } + /** @} */ @@ -411,6 +660,34 @@ fman_get_bushandle(device_t dev, vm_offset_t *fm_base) return (0); } +int +fman_get_pcd_handle(device_t dev, t_Handle *pcdh) +{ + struct fman_softc *sc = device_get_softc(dev); + + *pcdh = sc->pcd_handle; + + return (0); +} + +int +fman_get_netenv_handle(device_t dev, t_Handle *netenvh) +{ + struct fman_softc *sc = device_get_softc(dev); + + *netenvh = sc->netenv_handle; + + return (0); +} + +struct fmcd_softc * +fman_get_fmcd(device_t dev) +{ + struct fman_softc *sc = device_get_softc(dev); + + return (sc->fmcd); +} + int fman_attach(device_t dev) { @@ -475,7 +752,20 @@ fman_attach(device_t dev) goto err; } - return (bus_generic_attach(dev)); + /* Initialize FMD chardev interface for userspace PCD control */ + if (sc->pcd_handle != NULL) { + sc->fmcd = malloc(sizeof(struct fmcd_softc), M_FMAN, + M_WAITOK | M_ZERO); + if (fmcd_init(dev, sc->fmcd) != 0) { + free(sc->fmcd, M_FMAN); + sc->fmcd = NULL; + device_printf(dev, + "fmcd: chardev init failed (non-fatal)\n"); + } + } + + bus_attach_children(dev); + return (0); err: fman_detach(dev); @@ -489,12 +779,29 @@ fman_detach(device_t dev) sc = device_get_softc(dev); - if (sc->muram_handle) { - FM_MURAM_Free(sc->muram_handle); + if (sc->fmcd) { + fmcd_destroy(sc->fmcd); + free(sc->fmcd, M_FMAN); + sc->fmcd = NULL; + } + if (sc->netenv_handle) { + FM_PCD_NetEnvCharacteristicsDelete(sc->netenv_handle); + sc->netenv_handle = NULL; + } + if (sc->pcd_handle) { + FM_PCD_Disable(sc->pcd_handle); + FM_PCD_Free(sc->pcd_handle); + sc->pcd_handle = NULL; } if (sc->fm_handle) { FM_Free(sc->fm_handle); + sc->fm_handle = NULL; + } + + if (sc->muram_handle) { + FM_MURAM_Free(sc->muram_handle); + sc->muram_handle = NULL; } if (sc->mem_res) { @@ -507,7 +814,7 @@ fman_detach(device_t dev) sc->irq_res); } - if (sc->irq_res) { + if (sc->err_irq_res) { bus_release_resource(dev, SYS_RES_IRQ, sc->err_irq_rid, sc->err_irq_res); } @@ -540,8 +847,15 @@ int fman_qman_channel_id(device_t dev, int port) { struct fman_softc *sc; +#ifdef __aarch64__ + /* FMan v3 / Major >= 6 (LS1046A): 2x 10G + 6x 1G + 6x OH ports. + * Order matches Linux fman_get_qman_channel_id() for Major >= 6. */ + int qman_port_id[] = {0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b, + 0x2c, 0x2d, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}; +#else int qman_port_id[] = {0x31, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07}; +#endif int i; sc = device_get_softc(dev); @@ -553,4 +867,152 @@ fman_qman_channel_id(device_t dev, int port) return (0); } +/* + * Hot-add Host Command port to existing PCD for advanced offload. + * + * Called by CDX module init (after QMan portals are ready) to add HC + * support to the existing PCD. Unlike the previous approach of + * destroying and rebuilding PCD (which invalidated dtsec schemes and + * caused FM_FD_ERR_NO_SCHEME), this preserves the PCD, all existing + * KeyGen schemes, and dtsec port bindings. + * + * Sequence: + * 1. Create QMan FQRs for HC TX/confirmation/error + * 2. Disable PCD (brief pause — schemes survive) + * 3. Create HC port via FmHcConfigAndInit + * 4. Set p_FmPcd->h_Hc to enable HC in the PCD + * 5. FM_PCD_SetAdvancedOffloadSupport (requires HC + PCD disabled) + * 6. Re-enable PCD + * + * Returns 0 on success, errno on failure. + */ +int +fman_reinit_pcd_with_hc(device_t dev) +{ + struct fman_softc *sc; + t_FmPcd *p_FmPcd; + t_FmHcParams hc_params; + t_Handle hc_handle; + uint32_t hc_qman_chan; + t_Handle fqr; + t_Error error; + + sc = device_get_softc(dev); + + if (sc->pcd_handle == NULL) { + device_printf(dev, "PCD not initialized\n"); + return (ENXIO); + } + + /* Resolve QMan channel for HC offline port (cell-index 0x2) */ + hc_qman_chan = fman_qman_channel_id(dev, FMAN_HC_OH_CELL_INDEX); + if (hc_qman_chan == 0) { + device_printf(dev, + "couldn't get QMan channel for HC port\n"); + return (ENXIO); + } + + /* TX FQR — HC enqueues command frames here for FMan */ + fqr = qman_fqr_create(1, hc_qman_chan, FMAN_HC_FQ_WQ, + false, 0, false, false, true, false, 0, 0, 0); + if (fqr == NULL) { + device_printf(dev, "couldn't create HC TX FQR\n"); + return (ENOMEM); + } + sc->hc_tx_fqr = fqr; + + /* Confirmation FQR — FMan sends HC completions here */ + fqr = qman_fqr_create(1, e_QM_FQ_CHANNEL_POOL1, + FMAN_HC_FQ_WQ, false, 0, false, false, true, false, + 0, 0, 0); + if (fqr == NULL) { + device_printf(dev, "couldn't create HC confirmation FQR\n"); + return (ENOMEM); + } + sc->hc_conf_fqr = fqr; + + /* + * Register callback for HC confirmation frames. + * FMan sends command completions on confFqid; without this + * callback, they hit QMan's dummy handler and EnQFrm() in + * hc.c spins until timeout. + */ + error = qman_fqr_register_cb(sc->hc_conf_fqr, + fman_hc_conf_callback, sc); + if (error != E_OK) { + device_printf(dev, + "couldn't register HC confirmation callback: %d\n", + error); + return (ENXIO); + } + + /* Error FQR — FMan sends HC errors here */ + fqr = qman_fqr_create(1, e_QM_FQ_CHANNEL_POOL1, + FMAN_HC_FQ_WQ, false, 0, false, false, true, false, + 0, 0, 0); + if (fqr == NULL) { + device_printf(dev, "couldn't create HC error FQR\n"); + return (ENOMEM); + } + sc->hc_err_fqr = fqr; + + /* + * Temporarily disable PCD to add HC and enable advanced + * offload. Existing KeyGen schemes and port bindings survive + * the Disable/Enable cycle — only the PRS/KG/PLCR hardware + * engines are paused. + */ + error = FM_PCD_Disable(sc->pcd_handle); + if (error != E_OK) { + device_printf(dev, "couldn't disable PCD: %d\n", error); + return (ENXIO); + } + + /* Create HC port and attach to existing PCD */ + memset(&hc_params, 0, sizeof(hc_params)); + hc_params.h_Fm = sc->fm_handle; + hc_params.h_FmPcd = sc->pcd_handle; + hc_params.params.portBaseAddr = rman_get_bushandle(sc->mem_res) + + FMAN_HC_OH_REG_OFFSET; + hc_params.params.portId = FMAN_HC_PORT_ID; + hc_params.params.liodnBase = FMAN_HC_LIODN_BASE; + hc_params.params.errFqid = + qman_fqr_get_base_fqid(sc->hc_err_fqr); + hc_params.params.confFqid = + qman_fqr_get_base_fqid(sc->hc_conf_fqr); + hc_params.params.qmChannel = hc_qman_chan; + hc_params.params.f_QmEnqueue = fman_hc_qm_enqueue; + hc_params.params.h_QmArg = sc; + + hc_handle = FmHcConfigAndInit(&hc_params); + if (hc_handle == NULL) { + device_printf(dev, "couldn't create HC port\n"); + FM_PCD_Enable(sc->pcd_handle); + return (ENXIO); + } + + /* Wire HC into the existing PCD */ + p_FmPcd = (t_FmPcd *)sc->pcd_handle; + p_FmPcd->h_Hc = hc_handle; + + /* Enable advanced offload (requires HC + PCD disabled) */ + error = FM_PCD_SetAdvancedOffloadSupport(sc->pcd_handle); + if (error != E_OK) { + device_printf(dev, + "couldn't set advanced offload support: %d\n", error); + /* Non-fatal — HC still works, just no offload flag */ + } + + /* Re-enable PCD — resumes Parser/KeyGen/CC processing */ + error = FM_PCD_Enable(sc->pcd_handle); + if (error != E_OK) { + device_printf(dev, "couldn't re-enable PCD: %d\n", error); + return (ENXIO); + } + + device_printf(dev, "PCD upgraded: HC port added, " + "existing schemes preserved\n"); + return (0); +} + /** @} */ diff --git a/sys/dev/dpaa/fman.h b/sys/dev/dpaa/fman.h index c7b57c60171d..affb2dc6a752 100644 --- a/sys/dev/dpaa/fman.h +++ b/sys/dev/dpaa/fman.h @@ -27,8 +27,11 @@ #ifndef FMAN_H_ #define FMAN_H_ +#include #include +struct fmcd_softc; /* Forward declaration */ + /** * FMan driver instance data. */ @@ -46,6 +49,16 @@ struct fman_softc { t_Handle fm_handle; t_Handle muram_handle; + t_Handle pcd_handle; + t_Handle netenv_handle; + + /* Host Command port state (for advanced offload) */ + t_Handle hc_tx_fqr; /* TX FQR for HC enqueue */ + t_Handle hc_err_fqr; /* Error FQR for HC port */ + t_Handle hc_conf_fqr; /* Confirmation FQR for HC port */ + + /* FMD chardev interface (for userspace PCD control via fmlib/FMC) */ + struct fmcd_softc *fmcd; }; @@ -73,5 +86,9 @@ uint32_t fman_get_clock(struct fman_softc *sc); int fman_get_handle(device_t dev, t_Handle *fmh); int fman_get_muram_handle(device_t dev, t_Handle *muramh); int fman_get_bushandle(device_t dev, vm_offset_t *fm_base); +int fman_get_pcd_handle(device_t dev, t_Handle *pcdh); +int fman_get_netenv_handle(device_t dev, t_Handle *netenvh); +int fman_reinit_pcd_with_hc(device_t dev); +struct fmcd_softc *fman_get_fmcd(device_t dev); #endif /* FMAN_H_ */ diff --git a/sys/dev/dpaa/fman_chardev.c b/sys/dev/dpaa/fman_chardev.c new file mode 100644 index 000000000000..a30ab3593712 --- /dev/null +++ b/sys/dev/dpaa/fman_chardev.c @@ -0,0 +1,1774 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * FMan character device — exposes NCSW PCD API to userspace via ioctls. + * + * Creates /dev/fm0, /dev/fm0-pcd, /dev/fm0-port-rx*, /dev/fm0-port-oh* + * character devices. Userspace tools (fmlib, FMC, dpa_app) open these devices + * and issue ioctls that map 1:1 to NCSW kernel functions: + * + * FM_PCD_IOC_KG_SCHEME_SET → FM_PCD_KgSchemeSet() + * FM_PCD_IOC_HASH_TABLE_SET → FM_PCD_HashTableSet() + * FM_PCD_IOC_CC_ROOT_BUILD → FM_PCD_CcRootBuild() + * FM_PORT_IOC_SET_PCD → FM_PORT_SetPCD() + * ... etc. + * + * Handle management: NCSW returns opaque kernel pointers (t_Handle). + * A per-device handle table maps these to opaque integer IDs that are + * exposed to userspace — kernel pointers are never leaked. The ioc + * structs have identical binary layouts to the NCSW structs (plus a + * trailing void *id for output handles); the kernel translates handle + * IDs to/from kernel pointers at the ioctl boundary. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "opt_dpaa.h" + +#include +#include +#include +#include +#include +#include + +/* + * fmlib ioctl ABI headers — authoritative struct/ioctl definitions. + * Located in contrib/ncsw/inc/ioctls/ (included via DPAA_COMPILE_CMD). + */ +#include "fm_pcd_ioctls.h" + +#include "fman.h" +#include "fman_chardev.h" + +static MALLOC_DEFINE(M_FMCD, "fmcd", "FMan chardev data"); + +/* + * Port PCD types and ioctl commands. + * + * These come from fm_port_ioctls.h in fmlib, but we cannot include that + * header because it pulls in fmlib's "enet_ext.h" which conflicts with + * the NCSW kernel copy (contrib/ncsw/inc/enet_ext.h). The types below + * are copied verbatim from fm_port_ioctls.h and must stay in sync. + */ +typedef enum ioc_fm_port_pcd_support { + e_IOC_FM_PORT_PCD_SUPPORT_NONE = 0, + e_IOC_FM_PORT_PCD_SUPPORT_PRS_ONLY, + e_IOC_FM_PORT_PCD_SUPPORT_PLCR_ONLY, + e_IOC_FM_PORT_PCD_SUPPORT_PRS_AND_PLCR, + e_IOC_FM_PORT_PCD_SUPPORT_PRS_AND_KG, + e_IOC_FM_PORT_PCD_SUPPORT_PRS_AND_KG_AND_CC, + e_IOC_FM_PORT_PCD_SUPPORT_PRS_AND_KG_AND_CC_AND_PLCR, + e_IOC_FM_PORT_PCD_SUPPORT_PRS_AND_KG_AND_PLCR, + e_IOC_FM_PORT_PCD_SUPPORT_PRS_AND_CC, + e_IOC_FM_PORT_PCD_SUPPORT_PRS_AND_CC_AND_PLCR, + e_IOC_FM_PORT_PCD_SUPPORT_CC_ONLY, +} ioc_fm_port_pcd_support; + +typedef union ioc_fm_pcd_hdr_prs_opts_u { + struct { + bool label_interpretation_enable; + ioc_net_header_type next_parse; + } mpls_prs_options; + struct { + uint16_t tag_protocol_id1; + uint16_t tag_protocol_id2; + } vlan_prs_options; + struct { + bool enable_mtu_check; + } pppoe_prs_options; + struct { + bool routing_hdr_disable; + } ipv6_prs_options; + struct { + bool pad_ignore_checksum; + } udp_prs_options; + struct { + bool pad_ignore_checksum; + } tcp_prs_options; +} ioc_fm_pcd_hdr_prs_opts_u; + +typedef struct ioc_fm_pcd_prs_additional_hdr_params_t { + ioc_net_header_type hdr; + bool err_disable; + bool soft_prs_enable; + uint8_t index_per_hdr; + bool use_prs_opts; + ioc_fm_pcd_hdr_prs_opts_u prs_opts; +} ioc_fm_pcd_prs_additional_hdr_params_t; + +typedef struct ioc_fm_port_pcd_prs_params_t { + uint8_t prs_res_priv_info; + uint8_t parsing_offset; + ioc_net_header_type first_prs_hdr; + bool include_in_prs_statistics; + uint8_t num_of_hdrs_with_additional_params; + ioc_fm_pcd_prs_additional_hdr_params_t + additional_params[IOC_FM_PCD_PRS_NUM_OF_HDRS]; + bool set_vlan_tpid1; + uint16_t vlan_tpid1; + bool set_vlan_tpid2; + uint16_t vlan_tpid2; +} ioc_fm_port_pcd_prs_params_t; + +typedef struct ioc_fm_port_pcd_cc_params_t { + void *cc_tree_id; +} ioc_fm_port_pcd_cc_params_t; + +typedef struct ioc_fm_port_pcd_kg_params_t { + uint8_t num_of_schemes; + void *scheme_ids[FM_PCD_KG_NUM_OF_SCHEMES]; + bool direct_scheme; + void *direct_scheme_id; +} ioc_fm_port_pcd_kg_params_t; + +typedef struct ioc_fm_port_pcd_plcr_params_t { + void *plcr_profile_id; +} ioc_fm_port_pcd_plcr_params_t; + +typedef struct ioc_fm_port_pcd_params_t { + ioc_fm_port_pcd_support pcd_support; + void *net_env_id; + ioc_fm_port_pcd_prs_params_t *p_prs_params; + ioc_fm_port_pcd_cc_params_t *p_cc_params; + ioc_fm_port_pcd_kg_params_t *p_kg_params; + ioc_fm_port_pcd_plcr_params_t *p_plcr_params; + void *p_ip_reassembly_manip; + void *p_capwap_reassembly_manip; +} ioc_fm_port_pcd_params_t; + +#define FM_PORT_IOC_DISABLE _IO(FM_IOC_TYPE_BASE, FM_PORT_IOC_NUM(1)) +#define FM_PORT_IOC_ENABLE _IO(FM_IOC_TYPE_BASE, FM_PORT_IOC_NUM(2)) +#define FM_PORT_IOC_SET_PCD \ + _IOW(FM_IOC_TYPE_BASE, FM_PORT_IOC_NUM(20), ioc_fm_port_pcd_params_t) +#define FM_PORT_IOC_DELETE_PCD _IO(FM_IOC_TYPE_BASE, FM_PORT_IOC_NUM(21)) + +/* ------------------------------------------------------------------ */ +/* Struct layout assertions */ +/* ------------------------------------------------------------------ */ + +/* + * The ioc structs must have identical binary layout to the corresponding + * NCSW structs, with only a trailing void *id output field appended. + * This is verified at compile time, matching Linux's ASSERT_COND pattern. + * + * roundup2() accounts for alignment padding the compiler may insert + * before the trailing void *id pointer (e.g. NetEnvParams: NCSW=964, + * ioc=976 because of 4 bytes padding to align void *id to 8 bytes). + */ +#define FMCD_IOC_SIZE(ncsw_t) \ + roundup2(sizeof(ncsw_t) + sizeof(void *), _Alignof(void *)) + +_Static_assert( + sizeof(ioc_fm_pcd_net_env_params_t) == FMCD_IOC_SIZE(t_FmPcdNetEnvParams), + "NetEnvParams size mismatch — ioc and NCSW struct layouts diverged"); + +_Static_assert( + sizeof(ioc_fm_pcd_kg_scheme_params_t) == + FMCD_IOC_SIZE(t_FmPcdKgSchemeParams), + "KgSchemeParams size mismatch — ioc and NCSW struct layouts diverged"); + +_Static_assert( + sizeof(ioc_fm_pcd_cc_tree_params_t) == + FMCD_IOC_SIZE(t_FmPcdCcTreeParams), + "CcTreeParams size mismatch — ioc and NCSW struct layouts diverged"); + +_Static_assert( + sizeof(ioc_fm_pcd_cc_node_params_t) == + FMCD_IOC_SIZE(t_FmPcdCcNodeParams), + "CcNodeParams size mismatch — ioc and NCSW struct layouts diverged"); + +_Static_assert( + sizeof(ioc_fm_pcd_hash_table_params_t) == + FMCD_IOC_SIZE(t_FmPcdHashTableParams), + "HashTableParams size mismatch — ioc and NCSW struct layouts diverged"); + +_Static_assert( + sizeof(ioc_fm_pcd_manip_params_t) == + FMCD_IOC_SIZE(t_FmPcdManipParams), + "ManipParams size mismatch — ioc and NCSW struct layouts diverged"); + +_Static_assert( + sizeof(ioc_fm_pcd_plcr_profile_params_t) == + FMCD_IOC_SIZE(t_FmPcdPlcrProfileParams), + "PlcrProfileParams size mismatch — ioc and NCSW struct layouts diverged"); + +#if (DPAA_VERSION >= 11) +_Static_assert( + sizeof(ioc_fm_pcd_frm_replic_group_params_t) == + FMCD_IOC_SIZE(t_FmPcdFrmReplicGroupParams), + "FrmReplicGroupParams size mismatch — ioc and NCSW struct layouts diverged"); +#endif + +/* Port PCD struct: no trailing id field, sizes must match exactly. */ +_Static_assert( + sizeof(ioc_fm_port_pcd_params_t) == sizeof(t_FmPortPcdParams), + "PortPcdParams size mismatch — ioc and NCSW struct layouts diverged"); + +/* ------------------------------------------------------------------ */ +/* Port registration */ +/* ------------------------------------------------------------------ */ + +/* + * Convert DT cell-index to fmlib RX port index. + * + * fmlib convention (matching Linux SDK): + * 1G MACs: cell-index 0-5 → port index 0-5 + * 10G MACs: cell-index 8-9 → port index 6-7 + */ +#define FMCD_1G_RX_PORTS 6 + +static inline int +fmcd_cell_to_rxidx(int cell_index) +{ + if (cell_index >= 8) /* 10G MAC */ + return (FMCD_1G_RX_PORTS + (cell_index - 8)); + return (cell_index); /* 1G MAC */ +} + +void +fmcd_register_rx_port(struct fmcd_softc *sc, int cell_index, t_Handle h_port, + uint32_t dflt_fqid) +{ + int index = fmcd_cell_to_rxidx(cell_index); + + if (sc == NULL || index < 0 || index >= 8) + return; + + sx_xlock(&sc->port_lock); + sc->rx_port_handles[index] = h_port; + sc->rx_port_fqids[index] = dflt_fqid; + if (sc->cdev_port_rx[index] != NULL) + sc->cdev_port_rx[index]->si_drv2 = h_port; + sx_xunlock(&sc->port_lock); + + device_printf(sc->fman_dev, + "fmcd: rx port registered: cell=%d idx=%d fqid=%u\n", + cell_index, index, dflt_fqid); +} + +uint32_t +fmcd_get_rx_dflt_fqid(struct fmcd_softc *sc, int index) +{ + + if (sc == NULL || index < 0 || index >= 8) + return (0); + return (sc->rx_port_fqids[index]); +} + +void +fmcd_register_rx_port_scheme(struct fmcd_softc *sc, int cell_index, + t_Handle h_scheme, t_Handle *p_scheme) +{ + int index = fmcd_cell_to_rxidx(cell_index); + + if (sc == NULL || index < 0 || index >= 8) + return; + + sx_xlock(&sc->port_lock); + sc->rx_port_schemes[index] = h_scheme; + sc->rx_port_scheme_ptrs[index] = p_scheme; + sx_xunlock(&sc->port_lock); +} + +void +fmcd_register_oh_port(struct fmcd_softc *sc, int index, t_Handle h_port) +{ + if (sc == NULL || index < 0 || index >= 6) + return; + + sx_xlock(&sc->port_lock); + sc->oh_port_handles[index] = h_port; + if (sc->cdev_port_oh[index] != NULL) + sc->cdev_port_oh[index]->si_drv2 = h_port; + sx_xunlock(&sc->port_lock); +} + +void +fmcd_pcd_mark_transferred(struct fmcd_softc *sc) +{ + + sx_xlock(&sc->port_lock); + sc->pcd_transferred = true; + sx_xunlock(&sc->port_lock); +} + +/* ------------------------------------------------------------------ */ +/* Handle table helpers */ +/* ------------------------------------------------------------------ */ + +/* + * Allocate a handle slot, store the kernel pointer and type. + * Returns the opaque handle ID, or 0 on failure (table full). + * Caller must hold sc->port_lock. + */ +static uint64_t +fmcd_handle_alloc(struct fmcd_softc *sc, t_Handle ptr, + enum fmcd_handle_type type) +{ + int i; + + KASSERT(ptr != NULL, ("fmcd_handle_alloc: NULL ptr")); + KASSERT(type != FMCD_HDL_NONE, ("fmcd_handle_alloc: NONE type")); + + for (i = 0; i < FMCD_MAX_HANDLES; i++) { + if (sc->handles[i].ptr == NULL) { + sc->handles[i].ptr = ptr; + sc->handles[i].type = type; + return (((uint64_t)sc->handle_gen << + FMCD_HANDLE_SHIFT) | (uint64_t)i); + } + } + printf("fmcd: handle table full (%d entries)\n", FMCD_MAX_HANDLES); + return (0); +} + +/* + * Look up a handle ID and return the kernel pointer. + * If expected_type is not FMCD_HDL_NONE, validates the type matches. + * Returns NULL if the handle is invalid, stale, or type mismatches. + * Caller must hold sc->port_lock. + */ +static t_Handle +fmcd_handle_lookup(struct fmcd_softc *sc, uint64_t id, + enum fmcd_handle_type expected_type) +{ + uint32_t slot, gen; + + if (id == 0) + return (NULL); + + slot = (uint32_t)(id & FMCD_HANDLE_MASK); + gen = (uint32_t)(id >> FMCD_HANDLE_SHIFT); + + if (slot >= FMCD_MAX_HANDLES) + return (NULL); + if (gen != sc->handle_gen) + return (NULL); + if (sc->handles[slot].ptr == NULL) + return (NULL); + if (expected_type != FMCD_HDL_NONE && + sc->handles[slot].type != expected_type) + return (NULL); + + return (sc->handles[slot].ptr); +} + +/* + * Free a handle slot. Returns the kernel pointer for the caller + * to pass to NCSW delete, or NULL if the handle is invalid. + * Caller must hold sc->port_lock. + */ +static t_Handle +fmcd_handle_free(struct fmcd_softc *sc, uint64_t id, + enum fmcd_handle_type expected_type) +{ + uint32_t slot, gen; + t_Handle ptr; + + if (id == 0) + return (NULL); + + slot = (uint32_t)(id & FMCD_HANDLE_MASK); + gen = (uint32_t)(id >> FMCD_HANDLE_SHIFT); + + if (slot >= FMCD_MAX_HANDLES) + return (NULL); + if (gen != sc->handle_gen) + return (NULL); + if (sc->handles[slot].ptr == NULL) + return (NULL); + if (expected_type != FMCD_HDL_NONE && + sc->handles[slot].type != expected_type) + return (NULL); + + ptr = sc->handles[slot].ptr; + sc->handles[slot].ptr = NULL; + sc->handles[slot].type = FMCD_HDL_NONE; + return (ptr); +} + +/* + * Resolve a handle ID to a kernel pointer without freeing the slot. + * Used by the CDX module to translate hash table handles received + * from userspace (dpa_app) into NCSW kernel pointers. + * Returns NULL if the handle is invalid or stale. + */ +t_Handle +fmcd_handle_resolve(struct fmcd_softc *sc, uint64_t id, + enum fmcd_handle_type expected_type) +{ + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_lookup(sc, id, expected_type); + sx_xunlock(&sc->port_lock); + return (ptr); +} + +/* + * Translate a void* field that holds an opaque handle ID to a kernel + * pointer. Returns 0 on success (field updated in-place), EINVAL on + * bad handle. NULL fields are left alone (returns 0). + * Caller must hold sc->port_lock. + */ +static int +fmcd_xlate(struct fmcd_softc *sc, void **field, + enum fmcd_handle_type expected_type) +{ + t_Handle ptr; + + if (*field == NULL) + return (0); + + ptr = fmcd_handle_lookup(sc, (uint64_t)(uintptr_t)*field, + expected_type); + if (ptr == NULL) + return (EINVAL); + + *field = ptr; + return (0); +} + +/* + * Translate handle IDs embedded within a cc_next_engine_params_t + * to kernel pointers. Returns 0 on success, EINVAL on bad handle. + * Caller must hold sc->port_lock. + */ +static int +fmcd_translate_next_engine(struct fmcd_softc *sc, + ioc_fm_pcd_cc_next_engine_params_t *ne) +{ + int error; + + switch (ne->next_engine) { + case e_IOC_FM_PCD_HASH: + case e_IOC_FM_PCD_CC: + error = fmcd_xlate(sc, + &ne->params.cc_params.cc_node_id, FMCD_HDL_NONE); + if (error) + return (error); + break; + case e_IOC_FM_PCD_KG: + error = fmcd_xlate(sc, + &ne->params.kg_params.p_direct_scheme, + FMCD_HDL_KG_SCHEME); + if (error) + return (error); + break; +#if (DPAA_VERSION >= 11) + case e_IOC_FM_PCD_FR: + error = fmcd_xlate(sc, + &ne->params.fr_params.frm_replic_id, + FMCD_HDL_FRM_REPLIC); + if (error) + return (error); + break; +#endif + default: + break; + } + + /* manip_id can appear regardless of next_engine */ + error = fmcd_xlate(sc, &ne->manip_id, FMCD_HDL_MANIP); + return (error); +} + +/* + * Translate handle IDs in a policer profile's color action params. + * Caller must hold sc->port_lock. + */ +static int +fmcd_translate_plcr_color(struct fmcd_softc *sc, + ioc_fm_pcd_engine next_engine, + ioc_fm_pcd_plcr_next_engine_params_u *p) +{ + int error; + + switch (next_engine) { + case e_IOC_FM_PCD_PLCR: + error = fmcd_xlate(sc, &p->p_profile, + FMCD_HDL_PLCR_PROFILE); + return (error); + case e_IOC_FM_PCD_KG: + error = fmcd_xlate(sc, &p->p_direct_scheme, + FMCD_HDL_KG_SCHEME); + return (error); + default: + return (0); + } +} + +/* ------------------------------------------------------------------ */ +/* PCD ioctl handlers */ +/* ------------------------------------------------------------------ */ + +static int +fmcd_ioc_pcd_enable(struct fmcd_softc *sc) +{ + t_Error err; + + err = FM_PCD_Enable(sc->pcd_handle); + return (err == E_OK ? 0 : EIO); +} + +static int +fmcd_ioc_pcd_disable(struct fmcd_softc *sc) +{ + t_Error err; + + err = FM_PCD_Disable(sc->pcd_handle); + return (err == E_OK ? 0 : EIO); +} + +static int +fmcd_ioc_get_api_version(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_api_version_t *p = (ioc_fm_api_version_t *)data; + + p->version.major = FMD_API_VERSION_MAJOR; + p->version.minor = FMD_API_VERSION_MINOR; + p->version.respin = FMD_API_VERSION_RESPIN; + p->version.reserved = 0; + return (0); +} + +/* + * NetEnvCharacteristicsSet — _IOWR, framework copies in/out. + * Direct cast: ioc struct layout == NCSW struct layout. + */ +static int +fmcd_ioc_net_env_set(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_net_env_params_t *param = + (ioc_fm_pcd_net_env_params_t *)data; + t_Handle ptr; + uint64_t id; + + ptr = FM_PCD_NetEnvCharacteristicsSet(sc->pcd_handle, + (t_FmPcdNetEnvParams *)param); + if (ptr == NULL) + return (EIO); + + sx_xlock(&sc->port_lock); + id = fmcd_handle_alloc(sc, ptr, FMCD_HDL_NET_ENV); + sx_xunlock(&sc->port_lock); + if (id == 0) { + FM_PCD_NetEnvCharacteristicsDelete(ptr); + return (ENOMEM); + } + + param->id = (void *)(uintptr_t)id; + return (0); +} + +static int +fmcd_ioc_net_env_delete(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_obj_t *ioc = (ioc_fm_obj_t *)data; + uint64_t id = (uint64_t)(uintptr_t)ioc->obj; + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_free(sc, id, FMCD_HDL_NET_ENV); + sx_xunlock(&sc->port_lock); + + if (ptr == NULL) + return (EINVAL); + + return (FM_PCD_NetEnvCharacteristicsDelete(ptr) + == E_OK ? 0 : EIO); +} + +/* + * KgSchemeSet — _IOWR, framework copies in/out. + * Direct cast: ioc struct layout == NCSW struct layout + * (after adding bool shared to t_FmPcdKgSchemeParams). + */ +static int +fmcd_ioc_kg_scheme_set(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_kg_scheme_params_t *param = + (ioc_fm_pcd_kg_scheme_params_t *)data; + t_FmPcdKgSchemeParams *ncsw = (t_FmPcdKgSchemeParams *)param; + t_Handle ptr; + uint64_t id; + int error; + + /* Translate input handle references */ + sx_xlock(&sc->port_lock); + if (param->modify) { + error = fmcd_xlate(sc, ¶m->scm_id.scheme_id, + FMCD_HDL_KG_SCHEME); + if (error) + goto unlock; + } + error = fmcd_xlate(sc, ¶m->net_env_params.net_env_id, + FMCD_HDL_NET_ENV); + if (error) + goto unlock; + if (param->next_engine == e_IOC_FM_PCD_CC) { + error = fmcd_xlate(sc, + ¶m->kg_next_engine_params.cc.tree_id, + FMCD_HDL_CC_ROOT); + if (error) + goto unlock; + } + /* PLCR profile in KG scheme context uses a numeric index + * (profile_select), not a handle — no translation needed. */ + sx_xunlock(&sc->port_lock); + + ptr = FM_PCD_KgSchemeSet(sc->pcd_handle, ncsw); + if (ptr == NULL) + return (EIO); + + sx_xlock(&sc->port_lock); + id = fmcd_handle_alloc(sc, ptr, FMCD_HDL_KG_SCHEME); + sx_xunlock(&sc->port_lock); + if (id == 0) { + FM_PCD_KgSchemeDelete(ptr); + return (ENOMEM); + } + + param->id = (void *)(uintptr_t)id; + return (0); + +unlock: + sx_xunlock(&sc->port_lock); + return (error); +} + +static int +fmcd_ioc_kg_scheme_delete(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_obj_t *ioc = (ioc_fm_obj_t *)data; + uint64_t id = (uint64_t)(uintptr_t)ioc->obj; + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_free(sc, id, FMCD_HDL_KG_SCHEME); + sx_xunlock(&sc->port_lock); + + if (ptr == NULL) + return (EINVAL); + + return (FM_PCD_KgSchemeDelete(ptr) == E_OK ? 0 : EIO); +} + +/* + * CcRootBuild — _IO on FreeBSD (struct exceeds IOCPARM_MAX). + * Manual copyin/copyout, direct cast. + */ +static int +fmcd_ioc_cc_root_build(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_cc_tree_params_t *param; + void *uptr; + t_Handle ptr; + uint64_t hid; + int i, j, error; + + uptr = *(void **)data; + if (uptr == NULL) + return (EINVAL); + + param = malloc(sizeof(*param), M_FMCD, M_WAITOK); + error = copyin(uptr, param, sizeof(*param)); + if (error) { + free(param, M_FMCD); + return (error); + } + + /* Translate input handle references */ + sx_xlock(&sc->port_lock); + error = fmcd_xlate(sc, ¶m->net_env_id, FMCD_HDL_NET_ENV); + if (error) + goto unlock; + + for (i = 0; i < param->num_of_groups && + i < IOC_FM_PCD_MAX_NUM_OF_CC_GROUPS; i++) { + uint8_t ndu = param->fm_pcd_cc_group_params[i]. + num_of_distinction_units; + int n_entries = (1 << ndu); /* 2^ndu entries per group */ + for (j = 0; j < n_entries && + j < IOC_FM_PCD_MAX_NUM_OF_CC_ENTRIES_IN_GRP; j++) { + ioc_fm_pcd_cc_next_engine_params_t *ne = + ¶m->fm_pcd_cc_group_params[i]. + next_engine_per_entries_in_grp[j]; + error = fmcd_translate_next_engine(sc, ne); + if (error) + goto unlock; + } + } + sx_xunlock(&sc->port_lock); + + ptr = FM_PCD_CcRootBuild(sc->pcd_handle, + (t_FmPcdCcTreeParams *)param); + if (ptr == NULL) { + free(param, M_FMCD); + return (EIO); + } + + sx_xlock(&sc->port_lock); + hid = fmcd_handle_alloc(sc, ptr, FMCD_HDL_CC_ROOT); + sx_xunlock(&sc->port_lock); + if (hid == 0) { + FM_PCD_CcRootDelete(ptr); + free(param, M_FMCD); + return (ENOMEM); + } + + param->id = (void *)(uintptr_t)hid; + error = copyout(param, uptr, sizeof(*param)); + free(param, M_FMCD); + return (error); + +unlock: + sx_xunlock(&sc->port_lock); + free(param, M_FMCD); + return (error); +} + +static int +fmcd_ioc_cc_root_delete(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_obj_t *ioc = (ioc_fm_obj_t *)data; + uint64_t id = (uint64_t)(uintptr_t)ioc->obj; + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_free(sc, id, FMCD_HDL_CC_ROOT); + sx_xunlock(&sc->port_lock); + + if (ptr == NULL) + return (EINVAL); + + return (FM_PCD_CcRootDelete(ptr) == E_OK ? 0 : EIO); +} + +/* + * HashTableSet — _IOWR, framework copies in/out. + * Direct cast. + */ +static int +fmcd_ioc_hash_table_set(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_hash_table_params_t *param = + (ioc_fm_pcd_hash_table_params_t *)data; + t_Handle ptr; + uint64_t id; + int error; + + /* Translate miss next-engine handle references */ + sx_xlock(&sc->port_lock); + error = fmcd_translate_next_engine(sc, + ¶m->cc_next_engine_params_for_miss); + sx_xunlock(&sc->port_lock); + if (error) + return (error); + + ptr = FM_PCD_HashTableSet(sc->pcd_handle, + (t_FmPcdHashTableParams *)param); + if (ptr == NULL) + return (EIO); + + sx_xlock(&sc->port_lock); + id = fmcd_handle_alloc(sc, ptr, FMCD_HDL_HASH_TABLE); + sx_xunlock(&sc->port_lock); + if (id == 0) { + FM_PCD_HashTableDelete(ptr); + return (ENOMEM); + } + + param->id = (void *)(uintptr_t)id; + return (0); +} + +static int +fmcd_ioc_hash_table_delete(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_obj_t *ioc = (ioc_fm_obj_t *)data; + uint64_t id = (uint64_t)(uintptr_t)ioc->obj; + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_free(sc, id, FMCD_HDL_HASH_TABLE); + sx_xunlock(&sc->port_lock); + + if (ptr == NULL) + return (EINVAL); + + return (FM_PCD_HashTableDelete(ptr) == E_OK ? 0 : EIO); +} + +/* + * MatchTableSet — _IO on FreeBSD (struct exceeds IOCPARM_MAX). + * Manual copyin/copyout, direct cast. + * + * Special handling: key_params[i].p_key and p_mask point to userspace + * memory after copyin of the outer struct. Must copyin each key/mask + * buffer and redirect the pointers before calling NCSW. + */ +static int +fmcd_ioc_match_table_set(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_cc_node_params_t *param; + void *uptr; + uint8_t key_size; + int i, error; + int processed_keys = 0; + + uptr = *(void **)data; + if (uptr == NULL) + return (EINVAL); + + param = malloc(sizeof(*param), M_FMCD, M_WAITOK); + error = copyin(uptr, param, sizeof(*param)); + if (error) { + free(param, M_FMCD); + return (error); + } + + key_size = param->keys_params.key_size; + + /* + * Copyin key and mask buffers from userspace. + * After copyin of the outer struct, p_key/p_mask still reference + * userspace addresses. Allocate kernel buffers, copyin, redirect. + * + * Track processed_keys so the cleanup path only frees keys whose + * pointers were replaced with kernel allocations. Keys beyond + * processed_keys still hold raw userspace addresses from the + * outer copyin and must NOT be freed. + */ + for (i = 0; i < param->keys_params.num_of_keys && + i < IOC_FM_PCD_MAX_NUM_OF_KEYS; i++) { + ioc_fm_pcd_cc_key_params_t *kp = + ¶m->keys_params.key_params[i]; + void *umask = kp->p_mask; + + /* + * Clear p_mask before processing p_key so that if + * p_key copyin fails, the cleanup path won't call + * free() on the userspace address still in p_mask. + */ + kp->p_mask = NULL; + + if (kp->p_key != NULL && key_size > 0) { + void *usrc = kp->p_key; + kp->p_key = malloc(key_size, M_FMCD, M_WAITOK); + processed_keys = i + 1; + error = copyin(usrc, kp->p_key, key_size); + if (error) + goto fail; + } + + if (umask != NULL && key_size > 0) { + kp->p_mask = malloc(key_size, M_FMCD, M_WAITOK); + processed_keys = i + 1; + error = copyin(umask, kp->p_mask, key_size); + if (error) + goto fail; + } + + processed_keys = i + 1; + } + + /* Translate handle references in next-engine params */ + sx_xlock(&sc->port_lock); + for (i = 0; i < param->keys_params.num_of_keys && + i < IOC_FM_PCD_MAX_NUM_OF_KEYS; i++) { + error = fmcd_translate_next_engine(sc, + ¶m->keys_params.key_params[i].cc_next_engine_params); + if (error) { + sx_xunlock(&sc->port_lock); + goto fail; + } + } + error = fmcd_translate_next_engine(sc, + ¶m->keys_params.cc_next_engine_params_for_miss); + sx_xunlock(&sc->port_lock); + if (error) + goto fail; + + { + t_Handle ptr; + uint64_t hid; + + ptr = FM_PCD_MatchTableSet(sc->pcd_handle, + (t_FmPcdCcNodeParams *)param); + if (ptr == NULL) { + error = EIO; + goto fail; + } + + sx_xlock(&sc->port_lock); + hid = fmcd_handle_alloc(sc, ptr, FMCD_HDL_MATCH_TABLE); + sx_xunlock(&sc->port_lock); + if (hid == 0) { + FM_PCD_MatchTableDelete(ptr); + error = ENOMEM; + goto fail; + } + + param->id = (void *)(uintptr_t)hid; + } + + error = copyout(param, uptr, sizeof(*param)); + +fail: + /* + * Free only key/mask buffers that were replaced with kernel + * allocations (indices 0..processed_keys-1). Entries beyond + * processed_keys still hold userspace addresses and must not + * be passed to free(). + */ + for (i = 0; i < processed_keys; i++) { + ioc_fm_pcd_cc_key_params_t *kp = + ¶m->keys_params.key_params[i]; + if (kp->p_key != NULL) + free(kp->p_key, M_FMCD); + if (kp->p_mask != NULL) + free(kp->p_mask, M_FMCD); + } + free(param, M_FMCD); + return (error); +} + +static int +fmcd_ioc_match_table_delete(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_obj_t *ioc = (ioc_fm_obj_t *)data; + uint64_t id = (uint64_t)(uintptr_t)ioc->obj; + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_free(sc, id, FMCD_HDL_MATCH_TABLE); + sx_xunlock(&sc->port_lock); + + if (ptr == NULL) + return (EINVAL); + + return (FM_PCD_MatchTableDelete(ptr) == E_OK ? 0 : EIO); +} + +/* + * ManipNodeSet — _IOWR, framework copies in/out. + * Direct cast. + */ +static int +fmcd_ioc_manip_node_set(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_manip_params_t *param = + (ioc_fm_pcd_manip_params_t *)data; + t_Handle ptr; + uint64_t id; + int error; + + /* Translate chained manipulation handle */ + sx_xlock(&sc->port_lock); + error = fmcd_xlate(sc, ¶m->p_next_manip, FMCD_HDL_MANIP); + sx_xunlock(&sc->port_lock); + if (error) + return (error); + + ptr = FM_PCD_ManipNodeSet(sc->pcd_handle, + (t_FmPcdManipParams *)param); + if (ptr == NULL) + return (EIO); + + sx_xlock(&sc->port_lock); + id = fmcd_handle_alloc(sc, ptr, FMCD_HDL_MANIP); + sx_xunlock(&sc->port_lock); + if (id == 0) { + FM_PCD_ManipNodeDelete(ptr); + return (ENOMEM); + } + + param->id = (void *)(uintptr_t)id; + return (0); +} + +static int +fmcd_ioc_manip_node_delete(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_obj_t *ioc = (ioc_fm_obj_t *)data; + uint64_t id = (uint64_t)(uintptr_t)ioc->obj; + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_free(sc, id, FMCD_HDL_MANIP); + sx_xunlock(&sc->port_lock); + + if (ptr == NULL) + return (EINVAL); + + return (FM_PCD_ManipNodeDelete(ptr) == E_OK ? 0 : EIO); +} + +#if (DPAA_VERSION >= 11) +/* + * FrmReplicSetGroup — _IO on FreeBSD (struct exceeds IOCPARM_MAX). + * Manual copyin/copyout, direct cast. + */ +static int +fmcd_ioc_frm_replic_set_group(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_frm_replic_group_params_t *param; + void *uptr; + t_Handle ptr; + uint64_t hid; + int i, error; + + uptr = *(void **)data; + if (uptr == NULL) + return (EINVAL); + + param = malloc(sizeof(*param), M_FMCD, M_WAITOK); + error = copyin(uptr, param, sizeof(*param)); + if (error) { + free(param, M_FMCD); + return (error); + } + + /* Translate handle references in each entry's next-engine params */ + sx_xlock(&sc->port_lock); + for (i = 0; i < param->num_of_entries && + i < IOC_FM_PCD_FRM_REPLIC_MAX_NUM_OF_ENTRIES; i++) { + error = fmcd_translate_next_engine(sc, + ¶m->next_engine_params[i]); + if (error) { + sx_xunlock(&sc->port_lock); + free(param, M_FMCD); + return (error); + } + } + sx_xunlock(&sc->port_lock); + + ptr = FM_PCD_FrmReplicSetGroup(sc->pcd_handle, + (t_FmPcdFrmReplicGroupParams *)param); + if (ptr == NULL) { + free(param, M_FMCD); + return (EIO); + } + + sx_xlock(&sc->port_lock); + hid = fmcd_handle_alloc(sc, ptr, FMCD_HDL_FRM_REPLIC); + sx_xunlock(&sc->port_lock); + if (hid == 0) { + FM_PCD_FrmReplicDeleteGroup(ptr); + free(param, M_FMCD); + return (ENOMEM); + } + + param->id = (void *)(uintptr_t)hid; + error = copyout(param, uptr, sizeof(*param)); + free(param, M_FMCD); + return (error); +} + +static int +fmcd_ioc_frm_replic_delete_group(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_obj_t *ioc = (ioc_fm_obj_t *)data; + uint64_t id = (uint64_t)(uintptr_t)ioc->obj; + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_free(sc, id, FMCD_HDL_FRM_REPLIC); + sx_xunlock(&sc->port_lock); + + if (ptr == NULL) + return (EINVAL); + + return (FM_PCD_FrmReplicDeleteGroup(ptr) == E_OK ? 0 : EIO); +} +#endif /* DPAA_VERSION >= 11 */ + +/* + * PrsLoadSw — _IOW, framework copies in. + * Direct cast, but p_code pointer needs copyin + redirect. + */ +static int +fmcd_ioc_prs_load_sw(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_prs_sw_params_t *param = + (ioc_fm_pcd_prs_sw_params_t *)data; + uint8_t *code_buf; + t_Error err; + + if (param->size == 0 || param->p_code == NULL) + return (EINVAL); + if (param->size > FM_PCD_SW_PRS_SIZE) + return (EINVAL); + + /* Copyin the parser microcode from userspace */ + code_buf = malloc(param->size, M_FMCD, M_WAITOK); + if (copyin(param->p_code, code_buf, param->size) != 0) { + free(code_buf, M_FMCD); + return (EFAULT); + } + + /* Redirect pointer to kernel buffer */ + param->p_code = code_buf; + + err = FM_PCD_PrsLoadSw(sc->pcd_handle, (t_FmPcdPrsSwParams *)param); + free(code_buf, M_FMCD); + + return (err == E_OK ? 0 : EIO); +} + +/* + * PlcrProfileSet — _IOWR, framework copies in/out. + * Direct cast. + */ +static int +fmcd_ioc_plcr_profile_set(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_pcd_plcr_profile_params_t *param = + (ioc_fm_pcd_plcr_profile_params_t *)data; + t_Handle ptr; + uint64_t id; + int error; + + /* Translate input handle references */ + sx_xlock(&sc->port_lock); + if (param->modify) { + error = fmcd_xlate(sc, ¶m->profile_select.p_profile, + FMCD_HDL_PLCR_PROFILE); + if (error) + goto unlock; + } + error = fmcd_translate_plcr_color(sc, param->next_engine_on_green, + ¶m->params_on_green); + if (error) + goto unlock; + error = fmcd_translate_plcr_color(sc, param->next_engine_on_yellow, + ¶m->params_on_yellow); + if (error) + goto unlock; + error = fmcd_translate_plcr_color(sc, param->next_engine_on_red, + ¶m->params_on_red); + if (error) + goto unlock; + sx_xunlock(&sc->port_lock); + + ptr = FM_PCD_PlcrProfileSet(sc->pcd_handle, + (t_FmPcdPlcrProfileParams *)param); + if (ptr == NULL) + return (EIO); + + sx_xlock(&sc->port_lock); + id = fmcd_handle_alloc(sc, ptr, FMCD_HDL_PLCR_PROFILE); + sx_xunlock(&sc->port_lock); + if (id == 0) { + FM_PCD_PlcrProfileDelete(ptr); + return (ENOMEM); + } + + param->id = (void *)(uintptr_t)id; + return (0); + +unlock: + sx_xunlock(&sc->port_lock); + return (error); +} + +static int +fmcd_ioc_plcr_profile_delete(struct fmcd_softc *sc, caddr_t data) +{ + ioc_fm_obj_t *ioc = (ioc_fm_obj_t *)data; + uint64_t id = (uint64_t)(uintptr_t)ioc->obj; + t_Handle ptr; + + sx_xlock(&sc->port_lock); + ptr = fmcd_handle_free(sc, id, FMCD_HDL_PLCR_PROFILE); + sx_xunlock(&sc->port_lock); + + if (ptr == NULL) + return (EINVAL); + + return (FM_PCD_PlcrProfileDelete(ptr) == E_OK ? 0 : EIO); +} + +/* ------------------------------------------------------------------ */ +/* Port ioctl handlers */ +/* ------------------------------------------------------------------ */ + +/* + * PortSetPCD — _IOW, framework copies in the outer struct. + * The pointer members (p_prs_params, p_cc_params, p_kg_params, + * p_plcr_params) still reference userspace sub-structs after copyin. + * Copyin each and redirect, then direct-cast to NCSW type. + * + * Handle references (net_env_id, cc_tree_id, scheme_ids[], etc.) + * are translated from opaque IDs to kernel pointers. + */ +static int +fmcd_ioc_port_set_pcd(struct fmcd_softc *sc, t_Handle h_port, caddr_t data) +{ + ioc_fm_port_pcd_params_t *param = (ioc_fm_port_pcd_params_t *)data; + ioc_fm_port_pcd_prs_params_t *prs = NULL; + ioc_fm_port_pcd_cc_params_t *cc_p = NULL; + ioc_fm_port_pcd_kg_params_t *kg_p = NULL; + ioc_fm_port_pcd_plcr_params_t *plcr = NULL; + t_Error err; + int i, error = 0; + + if (h_port == NULL) + return (ENXIO); + + /* Copyin sub-structs from userspace pointers */ + if (param->p_prs_params != NULL) { + prs = malloc(sizeof(*prs), M_FMCD, M_WAITOK); + error = copyin(param->p_prs_params, prs, sizeof(*prs)); + if (error) + goto out; + param->p_prs_params = prs; + } + + if (param->p_cc_params != NULL) { + cc_p = malloc(sizeof(*cc_p), M_FMCD, M_WAITOK); + error = copyin(param->p_cc_params, cc_p, sizeof(*cc_p)); + if (error) + goto out; + param->p_cc_params = cc_p; + } + + if (param->p_kg_params != NULL) { + kg_p = malloc(sizeof(*kg_p), M_FMCD, M_WAITOK); + error = copyin(param->p_kg_params, kg_p, sizeof(*kg_p)); + if (error) + goto out; + param->p_kg_params = kg_p; + } + + if (param->p_plcr_params != NULL) { + plcr = malloc(sizeof(*plcr), M_FMCD, M_WAITOK); + error = copyin(param->p_plcr_params, plcr, sizeof(*plcr)); + if (error) + goto out; + param->p_plcr_params = plcr; + } + + /* Translate handle references in copyin'd sub-structs */ + sx_xlock(&sc->port_lock); + error = fmcd_xlate(sc, ¶m->net_env_id, FMCD_HDL_NET_ENV); + if (error) + goto unlock; + + if (cc_p != NULL) { + error = fmcd_xlate(sc, &cc_p->cc_tree_id, FMCD_HDL_CC_ROOT); + if (error) + goto unlock; + } + + if (kg_p != NULL) { + for (i = 0; i < kg_p->num_of_schemes && + i < FM_PCD_KG_NUM_OF_SCHEMES; i++) { + error = fmcd_xlate(sc, &kg_p->scheme_ids[i], + FMCD_HDL_KG_SCHEME); + if (error) + goto unlock; + } + if (kg_p->direct_scheme) { + error = fmcd_xlate(sc, &kg_p->direct_scheme_id, + FMCD_HDL_KG_SCHEME); + if (error) + goto unlock; + } + } + + if (plcr != NULL) { + error = fmcd_xlate(sc, &plcr->plcr_profile_id, + FMCD_HDL_PLCR_PROFILE); + if (error) + goto unlock; + } + + error = fmcd_xlate(sc, ¶m->p_ip_reassembly_manip, + FMCD_HDL_MANIP); + if (error) + goto unlock; + error = fmcd_xlate(sc, ¶m->p_capwap_reassembly_manip, + FMCD_HDL_MANIP); + if (error) + goto unlock; + sx_xunlock(&sc->port_lock); + + err = FM_PORT_SetPCD(h_port, (t_FmPortPcdParams *)param); + if (err != E_OK) { + printf("fmcd: SetPCD FAILED err=%d\n", (int)err); + error = EIO; + goto out; + } + +out: + free(prs, M_FMCD); + free(cc_p, M_FMCD); + free(kg_p, M_FMCD); + free(plcr, M_FMCD); + return (error); + +unlock: + sx_xunlock(&sc->port_lock); + goto out; +} + +static int +fmcd_ioc_port_delete_pcd(struct fmcd_softc *sc, t_Handle h_port) +{ + + if (h_port == NULL) + return (ENXIO); + + { + t_Error err = FM_PORT_DeletePCD(h_port); + return (err == E_OK ? 0 : EIO); + } +} + +static int +fmcd_ioc_port_enable(struct fmcd_softc *sc, t_Handle h_port) +{ + t_Error err; + + if (h_port == NULL) { + printf("fmcd: port_enable: h_port=NULL!\n"); + return (ENXIO); + } + + err = FM_PORT_Enable(h_port); + return (err == E_OK ? 0 : EIO); +} + +static int +fmcd_ioc_port_disable(struct fmcd_softc *sc, t_Handle h_port) +{ + t_Error err; + + if (h_port == NULL) { + printf("fmcd: port_disable: h_port=NULL!\n"); + return (ENXIO); + } + + err = FM_PORT_Disable(h_port); + return (err == E_OK ? 0 : EIO); +} + +/* ------------------------------------------------------------------ */ +/* Chardev operations */ +/* ------------------------------------------------------------------ */ + +static int +fmcd_open(struct cdev *dev, int flags, int fmt, struct thread *td) +{ + return (0); +} + +/* + * PCD device open handler. + * + * When userspace opens /dev/fm0-pcd, it is taking over PCD management + * (e.g., FMC/dpa_app is starting). Tear down any existing PCD set up + * by the dtsec driver (RSS KG schemes) to free KG scheme IDs for FMC. + * Also reset the handle table and bump the generation counter to + * invalidate any stale handles from a previous session. + */ +static int +fmcd_pcd_open(struct cdev *dev, int flags, int fmt, struct thread *td) +{ + struct fmcd_softc *sc = dev->si_drv1; + int i; + + sx_xlock(&sc->port_lock); + + for (i = 0; i < 8; i++) { + if (sc->rx_port_schemes[i] != NULL) { + if (sc->rx_port_handles[i] != NULL) + FM_PORT_DeletePCD(sc->rx_port_handles[i]); + FM_PCD_KgSchemeDelete(sc->rx_port_schemes[i]); + sc->rx_port_schemes[i] = NULL; + /* Clear dtsec's handle so pcd_free() won't + * double-free */ + if (sc->rx_port_scheme_ptrs[i] != NULL) { + *sc->rx_port_scheme_ptrs[i] = NULL; + sc->rx_port_scheme_ptrs[i] = NULL; + } + } + } + + /* + * Clear handle table and bump generation. We don't delete + * NCSW objects here — the previous session should have cleaned + * up via close, and the PCD enable/disable cycle that + * fmc/dpa_app does on startup resets PCD state. + */ + for (i = 0; i < FMCD_MAX_HANDLES; i++) { + sc->handles[i].ptr = NULL; + sc->handles[i].type = FMCD_HDL_NONE; + } + sc->handle_gen++; + if (sc->handle_gen == 0) + sc->handle_gen = 1; + + sx_xunlock(&sc->port_lock); + + return (0); +} + +/* + * PCD device close handler. + * + * Delete all PCD objects created during this session in dependency + * order (reverse of creation). This prevents kernel memory leaks + * if userspace crashes without explicit cleanup. + */ +static int +fmcd_pcd_close(struct cdev *dev, int flags, int fmt, struct thread *td) +{ + struct fmcd_softc *sc = dev->si_drv1; + int i; + + sx_xlock(&sc->port_lock); + + if (sc->pcd_transferred) { + /* + * CDX has taken ownership of all PCD resources via + * CDX_CTRL_DPA_SET_PARAMS. The NCSW objects (hash tables, + * CC trees, KG schemes, net envs) are bound to FMan ports + * and must not be deleted. Just clear the handle table. + */ + for (i = 0; i < FMCD_MAX_HANDLES; i++) { + sc->handles[i].ptr = NULL; + sc->handles[i].type = FMCD_HDL_NONE; + } + } else { + /* + * No CDX takeover — dpa_app exited before CDX claimed + * the PCD resources. Delete all NCSW objects in + * dependency order to prevent kernel memory leaks. + */ + static const enum fmcd_handle_type teardown_order[] = { + FMCD_HDL_FRM_REPLIC, + FMCD_HDL_PLCR_PROFILE, + FMCD_HDL_MATCH_TABLE, + FMCD_HDL_HASH_TABLE, + FMCD_HDL_CC_ROOT, + FMCD_HDL_KG_SCHEME, + FMCD_HDL_MANIP, + FMCD_HDL_NET_ENV, + }; + int phase; + + for (phase = 0; phase < nitems(teardown_order); phase++) { + for (i = 0; i < FMCD_MAX_HANDLES; i++) { + t_Handle ptr = sc->handles[i].ptr; + t_Error err; + + if (ptr == NULL || + sc->handles[i].type != + teardown_order[phase]) + continue; + + sc->handles[i].ptr = NULL; + sc->handles[i].type = FMCD_HDL_NONE; + + switch (teardown_order[phase]) { + case FMCD_HDL_NET_ENV: + err = FM_PCD_NetEnvCharacteristicsDelete(ptr); + break; + case FMCD_HDL_KG_SCHEME: + err = FM_PCD_KgSchemeDelete(ptr); + break; + case FMCD_HDL_CC_ROOT: + err = FM_PCD_CcRootDelete(ptr); + break; + case FMCD_HDL_MATCH_TABLE: + err = FM_PCD_MatchTableDelete(ptr); + break; + case FMCD_HDL_HASH_TABLE: + err = FM_PCD_HashTableDelete(ptr); + break; + case FMCD_HDL_MANIP: + err = FM_PCD_ManipNodeDelete(ptr); + break; + case FMCD_HDL_PLCR_PROFILE: + err = FM_PCD_PlcrProfileDelete(ptr); + break; + case FMCD_HDL_FRM_REPLIC: + err = FM_PCD_FrmReplicDeleteGroup(ptr); + break; + default: + err = E_OK; + break; + } + + if (err != E_OK) + printf("fmcd: close: failed to " + "delete slot %d type %d\n", + i, teardown_order[phase]); + } + } + } + + /* Bump generation to invalidate any lingering handle IDs */ + sc->handle_gen++; + if (sc->handle_gen == 0) + sc->handle_gen = 1; + + sc->pcd_transferred = false; + sx_xunlock(&sc->port_lock); + return (0); +} + +static int +fmcd_close(struct cdev *dev, int flags, int fmt, struct thread *td) +{ + return (0); +} + +/* + * PCD device ioctl handler (/dev/fm0-pcd). + */ +static int +fmcd_pcd_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, + struct thread *td) +{ + struct fmcd_softc *sc = dev->si_drv1; + int error; + + /* PCD operations require root */ + error = priv_check(td, PRIV_DRIVER); + if (error) + return (error); + + if (sc->pcd_handle == NULL) + return (ENXIO); + + switch (cmd) { + case FM_PCD_IOC_ENABLE: + return (fmcd_ioc_pcd_enable(sc)); + case FM_PCD_IOC_DISABLE: + return (fmcd_ioc_pcd_disable(sc)); + case FM_PCD_IOC_NET_ENV_CHARACTERISTICS_SET: + return (fmcd_ioc_net_env_set(sc, data)); + case FM_PCD_IOC_NET_ENV_CHARACTERISTICS_DELETE: + return (fmcd_ioc_net_env_delete(sc, data)); + case FM_PCD_IOC_KG_SCHEME_SET: + return (fmcd_ioc_kg_scheme_set(sc, data)); + case FM_PCD_IOC_KG_SCHEME_DELETE: + return (fmcd_ioc_kg_scheme_delete(sc, data)); + case FM_PCD_IOC_CC_ROOT_BUILD: + return (fmcd_ioc_cc_root_build(sc, data)); + case FM_PCD_IOC_CC_ROOT_DELETE: + return (fmcd_ioc_cc_root_delete(sc, data)); + case FM_PCD_IOC_HASH_TABLE_SET: + return (fmcd_ioc_hash_table_set(sc, data)); + case FM_PCD_IOC_HASH_TABLE_DELETE: + return (fmcd_ioc_hash_table_delete(sc, data)); + case FM_PCD_IOC_MATCH_TABLE_SET: + return (fmcd_ioc_match_table_set(sc, data)); + case FM_PCD_IOC_MATCH_TABLE_DELETE: + return (fmcd_ioc_match_table_delete(sc, data)); + case FM_PCD_IOC_MANIP_NODE_SET: + return (fmcd_ioc_manip_node_set(sc, data)); + case FM_PCD_IOC_MANIP_NODE_DELETE: + return (fmcd_ioc_manip_node_delete(sc, data)); +#if (DPAA_VERSION >= 11) + case FM_PCD_IOC_FRM_REPLIC_GROUP_SET: + return (fmcd_ioc_frm_replic_set_group(sc, data)); + case FM_PCD_IOC_FRM_REPLIC_GROUP_DELETE: + return (fmcd_ioc_frm_replic_delete_group(sc, data)); +#endif + case FM_PCD_IOC_PRS_LOAD_SW: + return (fmcd_ioc_prs_load_sw(sc, data)); + case FM_PCD_IOC_PLCR_PROFILE_SET: + return (fmcd_ioc_plcr_profile_set(sc, data)); + case FM_PCD_IOC_PLCR_PROFILE_DELETE: + return (fmcd_ioc_plcr_profile_delete(sc, data)); + case FM_PCD_IOC_SET_ADVANCED_OFFLOAD_SUPPORT: + return (FM_PCD_SetAdvancedOffloadSupport(sc->pcd_handle) + == E_OK ? 0 : EIO); + case FM_PCD_IOC_ALLOW_HC_USAGE: + /* HC is already configured by fman.c — just accept */ + return (0); + default: + return (ENOTTY); + } +} + +/* + * FM device ioctl handler (/dev/fm0). + */ +static int +fmcd_fm_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, + struct thread *td) +{ + struct fmcd_softc *sc = dev->si_drv1; + + switch (cmd) { + case FM_IOC_GET_API_VERSION: + return (fmcd_ioc_get_api_version(sc, data)); + default: + return (ENOTTY); + } +} + +/* + * Port device ioctl handler (/dev/fm0-port-rx*, /dev/fm0-port-oh*). + */ +static int +fmcd_port_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, + struct thread *td) +{ + struct fmcd_softc *sc = dev->si_drv1; + t_Handle h_port = dev->si_drv2; + int error; + + error = priv_check(td, PRIV_DRIVER); + if (error) + return (error); + + switch (cmd) { + case FM_PORT_IOC_SET_PCD: + return (fmcd_ioc_port_set_pcd(sc, h_port, data)); + case FM_PORT_IOC_DELETE_PCD: + return (fmcd_ioc_port_delete_pcd(sc, h_port)); + case FM_PORT_IOC_ENABLE: + return (fmcd_ioc_port_enable(sc, h_port)); + case FM_PORT_IOC_DISABLE: + return (fmcd_ioc_port_disable(sc, h_port)); + default: + return (ENOTTY); + } +} + +static struct cdevsw fmcd_fm_cdevsw = { + .d_version = D_VERSION, + .d_open = fmcd_open, + .d_close = fmcd_close, + .d_ioctl = fmcd_fm_ioctl, + .d_name = "fm", +}; + +static struct cdevsw fmcd_pcd_cdevsw = { + .d_version = D_VERSION, + .d_open = fmcd_pcd_open, + .d_close = fmcd_pcd_close, + .d_ioctl = fmcd_pcd_ioctl, + .d_name = "fm-pcd", +}; + +static struct cdevsw fmcd_port_cdevsw = { + .d_version = D_VERSION, + .d_open = fmcd_open, + .d_close = fmcd_close, + .d_ioctl = fmcd_port_ioctl, + .d_name = "fm-port", +}; + +/* ------------------------------------------------------------------ */ +/* Init / Destroy */ +/* ------------------------------------------------------------------ */ + +int +fmcd_init(device_t fman_dev, struct fmcd_softc *sc) +{ + struct fman_softc *fsc; + int i; + + fsc = device_get_softc(fman_dev); + + memset(sc, 0, sizeof(*sc)); + sc->fman_dev = fman_dev; + sc->fm_handle = fsc->fm_handle; + sc->pcd_handle = fsc->pcd_handle; + sc->handle_gen = 1; /* start at 1 so slot 0's first ID != 0 */ + + sx_init(&sc->port_lock, "fmcd_port"); + + /* /dev/fman0 */ + sc->cdev_fm = make_dev(&fmcd_fm_cdevsw, FMD_MINOR_FM, + UID_ROOT, GID_WHEEL, 0600, "fm%d", device_get_unit(fman_dev)); + if (sc->cdev_fm == NULL) { + device_printf(fman_dev, "fmcd: couldn't create /dev/fm\n"); + goto fail; + } + sc->cdev_fm->si_drv1 = sc; + + /* /dev/fman0-pcd */ + sc->cdev_pcd = make_dev(&fmcd_pcd_cdevsw, FMD_MINOR_PCD, + UID_ROOT, GID_WHEEL, 0600, "fm%d-pcd", + device_get_unit(fman_dev)); + if (sc->cdev_pcd == NULL) { + device_printf(fman_dev, + "fmcd: couldn't create /dev/fm-pcd\n"); + goto fail; + } + sc->cdev_pcd->si_drv1 = sc; + + /* + * Port devices are created eagerly — the handles are filled in + * when dtsec/oh drivers attach via fmcd_register_rx_port/oh_port. + */ + for (i = 0; i < 8; i++) { + sc->cdev_port_rx[i] = make_dev(&fmcd_port_cdevsw, + FMD_MINOR_RX_BASE + i, UID_ROOT, GID_WHEEL, 0600, + "fm%d-port-rx%d", device_get_unit(fman_dev), i); + if (sc->cdev_port_rx[i]) { + sc->cdev_port_rx[i]->si_drv1 = sc; + sc->cdev_port_rx[i]->si_drv2 = NULL; + } + } + + for (i = 0; i < 6; i++) { + sc->cdev_port_oh[i] = make_dev(&fmcd_port_cdevsw, + FMD_MINOR_OH_BASE + i, UID_ROOT, GID_WHEEL, 0600, + "fm%d-port-oh%d", device_get_unit(fman_dev), i); + if (sc->cdev_port_oh[i]) { + sc->cdev_port_oh[i]->si_drv1 = sc; + sc->cdev_port_oh[i]->si_drv2 = NULL; + } + } + + device_printf(fman_dev, "fmcd: chardev interface ready\n"); + return (0); + +fail: + fmcd_destroy(sc); + return (ENXIO); +} + +void +fmcd_destroy(struct fmcd_softc *sc) +{ + int i; + + for (i = 0; i < 6; i++) { + if (sc->cdev_port_oh[i]) + destroy_dev(sc->cdev_port_oh[i]); + } + for (i = 0; i < 8; i++) { + if (sc->cdev_port_rx[i]) + destroy_dev(sc->cdev_port_rx[i]); + } + if (sc->cdev_pcd) + destroy_dev(sc->cdev_pcd); + if (sc->cdev_fm) + destroy_dev(sc->cdev_fm); + + sx_destroy(&sc->port_lock); +} diff --git a/sys/dev/dpaa/fman_chardev.h b/sys/dev/dpaa/fman_chardev.h new file mode 100644 index 000000000000..523ba7ad8c02 --- /dev/null +++ b/sys/dev/dpaa/fman_chardev.h @@ -0,0 +1,155 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * FMan character device interface — exposes NCSW PCD API to userspace. + * + * Ioctl command numbers and data structures are ABI-compatible with the + * Linux NXP SDK FMD driver so that the same fmlib/FMC userspace tools + * work unchanged. + * + * Handle management: NCSW returns opaque kernel pointers (t_Handle). + * Userspace receives opaque integer handle IDs via a kernel handle + * table — kernel pointers are never exposed. The ioc structs have + * identical binary layouts to the NCSW structs (with a trailing + * void *id field); the kernel translates handle IDs to/from kernel + * pointers at the ioctl boundary. + * + * The ioctl struct definitions come from the fmlib ioctl headers + * (fm_pcd_ioctls.h, fm_port_ioctls.h) which are included directly + * by fman_chardev.c via the FMCD_COMPILE_CMD include paths. + * This header only exposes kernel-private definitions. + */ + +#ifndef FMAN_CHARDEV_H_ +#define FMAN_CHARDEV_H_ + +/* + * Device minor number layout. + * For FMan instance 0: + * minor 0 = /dev/fm0 (FM management) + * minor 1 = /dev/fm0-pcd (PCD operations) + * minor 2 = /dev/fm0-port-oh0 (OH port, cell-index 3) + * minor 3 = /dev/fm0-port-oh1 (OH port, cell-index 4) + * minor 8-15 = /dev/fm0-port-rx0..rx7 (RX ports, fmlib convention: + * 0-5 = 1G, 6-7 = 10G) + */ +#define FMD_MINOR_FM 0 +#define FMD_MINOR_PCD 1 +#define FMD_MINOR_OH_BASE 2 +#define FMD_MINOR_RX_BASE 8 /* after 6 OH slots */ +#define FMD_MINOR_MAX 16 /* 8 + 8 RX port slots */ + +/* ------------------------------------------------------------------ */ +/* Kernel-only: chardev state and functions */ +/* ------------------------------------------------------------------ */ + +#ifdef _KERNEL + +#include +#include +#include + +/* + * PCD handle table — maps opaque integer IDs (exposed to userspace) + * to kernel pointers (used internally by NCSW). + * + * Handle ID encoding (uint64_t, same width as void* on arm64): + * bits [6:0] = slot index (0..FMCD_MAX_HANDLES-1) + * bits [31:7] = generation counter (invalidates stale handles) + * bits [63:32] = 0 (ensures ID never resembles a kernel pointer) + * + * Value 0 is reserved as "invalid/NULL handle". handle_gen starts + * at 1 so slot 0's first ID is (1 << 7) | 0 = 128, not 0. + */ +enum fmcd_handle_type { + FMCD_HDL_NONE = 0, + FMCD_HDL_NET_ENV, + FMCD_HDL_KG_SCHEME, + FMCD_HDL_CC_ROOT, + FMCD_HDL_MATCH_TABLE, + FMCD_HDL_HASH_TABLE, + FMCD_HDL_MANIP, + FMCD_HDL_PLCR_PROFILE, + FMCD_HDL_FRM_REPLIC, +}; + +struct fmcd_handle_entry { + t_Handle ptr; /* kernel pointer (NULL = free slot) */ + enum fmcd_handle_type type; +}; + +#define FMCD_MAX_HANDLES 128 +#define FMCD_HANDLE_MASK 0x7f /* slot index in lower 7 bits */ +#define FMCD_HANDLE_SHIFT 7 /* generation counter above */ + +/* + * Per-FMan chardev state. + */ +struct fmcd_softc { + device_t fman_dev; /* parent fman device */ + struct cdev *cdev_fm; /* /dev/fm0 */ + struct cdev *cdev_pcd; /* /dev/fm0-pcd */ + struct cdev *cdev_port_rx[8]; /* /dev/fm0-port-rx[0-7] */ + struct cdev *cdev_port_oh[6]; /* /dev/fm0-port-oh[0-5] */ + + t_Handle fm_handle; /* NCSW FM handle */ + t_Handle pcd_handle; /* NCSW PCD handle */ + + /* Port handle registry — indexed by fmlib convention: + * 0-5 = 1G MACs (cell-index 0-5), 6-7 = 10G MACs (cell-index 8-9). + * fmcd_cell_to_rxidx() in fman_chardev.c converts DT cell-index. */ + struct sx port_lock; + t_Handle rx_port_handles[8]; + t_Handle rx_port_schemes[8]; /* dtsec KG scheme handles */ + t_Handle *rx_port_scheme_ptrs[8]; /* ptr to dtsec sc_scheme */ + uint32_t rx_port_fqids[8]; /* dtsec default RX FQIDs */ + t_Handle oh_port_handles[6]; + + /* PCD handle table — protected by port_lock */ + struct fmcd_handle_entry handles[FMCD_MAX_HANDLES]; + uint32_t handle_gen; /* generation counter */ + bool pcd_transferred; /* CDX has taken PCD ownership */ +}; + +/* + * Chardev lifecycle. + */ +int fmcd_init(device_t fman_dev, struct fmcd_softc *sc); +void fmcd_destroy(struct fmcd_softc *sc); + +/* + * Port registration (called by dtsec and dpaa_oh during attach). + */ +void fmcd_register_rx_port(struct fmcd_softc *sc, int index, + t_Handle h_port, uint32_t dflt_fqid); +void fmcd_register_oh_port(struct fmcd_softc *sc, int index, + t_Handle h_port); +uint32_t fmcd_get_rx_dflt_fqid(struct fmcd_softc *sc, int index); + +/* + * Register dtsec's KG scheme handle for PCD teardown. + * p_scheme points to dtsec_softc->sc_scheme so the chardev can + * clear it when userspace takes over PCD management. + */ +void fmcd_register_rx_port_scheme(struct fmcd_softc *sc, int index, + t_Handle h_scheme, t_Handle *p_scheme); + +/* + * Mark PCD resources as transferred to CDX. After this call, + * fmcd_pcd_close() will not attempt to delete NCSW PCD objects. + */ +void fmcd_pcd_mark_transferred(struct fmcd_softc *sc); + +/* + * Resolve a handle ID to a kernel pointer (read-only, does not free). + * Used by CDX to translate userspace handle IDs to NCSW pointers. + */ +t_Handle fmcd_handle_resolve(struct fmcd_softc *sc, uint64_t id, + enum fmcd_handle_type expected_type); + +#endif /* _KERNEL */ + +#endif /* FMAN_CHARDEV_H_ */ diff --git a/sys/dev/dpaa/fman_fdt.c b/sys/dev/dpaa/fman_fdt.c index 8fb06924e10a..5510f69b0e79 100644 --- a/sys/dev/dpaa/fman_fdt.c +++ b/sys/dev/dpaa/fman_fdt.c @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -62,8 +63,11 @@ static device_method_t fman_methods[] = { DEFINE_CLASS_1(fman, fman_driver, fman_methods, sizeof(struct fman_softc), simplebus_driver); +/* Probe after GIC (BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE = 45) + * so interrupt allocation works on the first attempt. + * Also after clockgen (BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE) for clk_get_freq(). */ EARLY_DRIVER_MODULE(fman, simplebus, fman_driver, 0, 0, - BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE); + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LAST); static int @@ -87,18 +91,29 @@ fman_get_clock(struct fman_softc *sc) device_t dev; phandle_t node; pcell_t fman_clock; + clk_t clk; + uint64_t freq; dev = sc->sc_base.dev; node = ofw_bus_get_node(dev); + /* Try static clock-frequency property first (PowerPC style) */ if ((OF_getprop(node, "clock-frequency", &fman_clock, - sizeof(fman_clock)) <= 0) || (fman_clock == 0)) { - device_printf(dev, "could not acquire correct frequency " - "from DTS\n"); - - return (0); + sizeof(fman_clock)) > 0) && (fman_clock != 0)) + return ((uint32_t)fman_clock); + + /* Fall back to clock framework (ARM64 style) */ + if (clk_get_by_ofw_index(dev, node, 0, &clk) == 0) { + if (clk_enable(clk) == 0 && + clk_get_freq(clk, &freq) == 0 && freq != 0) { + device_printf(dev, "FMan clock: %ju Hz\n", + (uintmax_t)freq); + return ((uint32_t)freq); + } + clk_release(clk); } - return ((uint32_t)fman_clock); + device_printf(dev, "could not acquire correct frequency from DTS\n"); + return (0); } diff --git a/sys/dev/dpaa/if_dtsec.c b/sys/dev/dpaa/if_dtsec.c index dfa76a476620..162673ef40d2 100644 --- a/sys/dev/dpaa/if_dtsec.c +++ b/sys/dev/dpaa/if_dtsec.c @@ -46,29 +46,50 @@ #include #include #include +#include #include #include #include #include #include +#include +#include #include "miibus_if.h" +#include "opt_dpaa.h" + #include #include #include +#include +#if (DPAA_VERSION < 11) +#include +#endif #include #include "fman.h" +#include "bman.h" + +/* FM port internal header — needed for RCCB register access in stall diag */ +#include +#include "qman.h" #include "if_dtsec.h" #include "if_dtsec_im.h" #include "if_dtsec_rm.h" +#include + #define DTSEC_MIN_FRAME_SIZE 64 #define DTSEC_MAX_FRAME_SIZE 9600 +#if (DPAA_VERSION < 11) #define DTSEC_REG_MAXFRM 0x110 +#define DTSEC_REG_GADDR(i) (0x0a0 + 4*(i)) +#else +#define MEMAC_REG_MAXFRM 0x014 +#endif /** * @group dTSEC private defines. @@ -156,7 +177,7 @@ dtsec_fm_mac_mdio_event_callback(t_Handle h_App, struct dtsec_softc *sc; sc = h_App; - device_printf(sc->sc_dev, "MDIO event %i: %s.\n", exception, + if_printf(sc->sc_ifnet, "MDIO event %i: %s.\n", exception, dtsec_fm_mac_ex_to_str(exception)); } @@ -166,7 +187,7 @@ dtsec_fm_mac_exception_callback(t_Handle app, e_FmMacExceptions exception) struct dtsec_softc *sc; sc = app; - device_printf(sc->sc_dev, "MAC exception %i: %s.\n", exception, + if_printf(sc->sc_ifnet, "MAC exception %i: %s.\n", exception, dtsec_fm_mac_ex_to_str(exception)); } @@ -214,7 +235,9 @@ dtsec_fm_mac_init(struct dtsec_softc *sc, uint8_t *mac) return (ENXIO); } - /* Do not inform about pause frames */ +#if (DPAA_VERSION < 11) + /* Do not inform about pause frames (dTSEC only; MEMAC has no + * equivalent interrupt — it simply never generates this event). */ error = FM_MAC_ConfigException(sc->sc_mach, e_FM_MAC_EX_1G_RX_CTL, FALSE); if (error != E_OK) { @@ -223,6 +246,7 @@ dtsec_fm_mac_init(struct dtsec_softc *sc, uint8_t *mac) dtsec_fm_mac_free(sc); return (ENXIO); } +#endif error = FM_MAC_Init(sc->sc_mach); if (error != E_OK) { @@ -232,6 +256,18 @@ dtsec_fm_mac_init(struct dtsec_softc *sc, uint8_t *mac) return (ENXIO); } + /* Diagnostic: dump MEMAC IF_MODE and IF_STATUS registers */ + { + uint32_t ifmode = be32toh(bus_read_4(sc->sc_mem, 0x300)); + uint32_t ifstat = be32toh(bus_read_4(sc->sc_mem, 0x304)); + device_printf(sc->sc_dev, + "MEMAC IF_MODE=0x%08x IF_STATUS=0x%08x mode=%s\n", + ifmode, ifstat, + (ifmode & 0x3) == 0 ? "XGMII(10G)" : + (ifmode & 0x3) == 2 ? "GMII(1G)" : + (ifmode & 0x3) == 4 ? "RGMII" : "unknown"); + } + return (0); } /** @} */ @@ -260,7 +296,7 @@ dtsec_fm_port_rx_exception_callback(t_Handle app, struct dtsec_softc *sc; sc = app; - device_printf(sc->sc_dev, "RX exception: %i: %s.\n", exception, + if_printf(sc->sc_ifnet, "RX exception: %i: %s.\n", exception, dtsec_fm_port_ex_to_str(exception)); } @@ -271,7 +307,7 @@ dtsec_fm_port_tx_exception_callback(t_Handle app, struct dtsec_softc *sc; sc = app; - device_printf(sc->sc_dev, "TX exception: %i: %s.\n", exception, + if_printf(sc->sc_ifnet, "TX exception: %i: %s.\n", exception, dtsec_fm_port_ex_to_str(exception)); } @@ -305,6 +341,11 @@ dtsec_fm_port_tx_type(enum eth_dev_type type) static void dtsec_fm_port_free_both(struct dtsec_softc *sc) { + if (sc->sc_vsph) { + FM_VSP_Free(sc->sc_vsph); + sc->sc_vsph = NULL; + } + if (sc->sc_rxph) { FM_PORT_Free(sc->sc_rxph); sc->sc_rxph = NULL; @@ -331,13 +372,52 @@ dtsec_set_mtu(struct dtsec_softc *sc, unsigned int mtu) DTSEC_LOCK_ASSERT(sc); if (mtu >= DTSEC_MIN_FRAME_SIZE && mtu <= DTSEC_MAX_FRAME_SIZE) { +#if (DPAA_VERSION < 11) bus_write_4(sc->sc_mem, DTSEC_REG_MAXFRM, mtu); +#else + bus_write_4(sc->sc_mem, MEMAC_REG_MAXFRM, + htobe32((uint32_t)mtu)); +#endif return (mtu); } return (0); } +static u_int +dtsec_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) +{ + struct dtsec_softc *sc = arg; + + FM_MAC_AddHashMacAddr(sc->sc_mach, (t_EnetAddr *)LLADDR(sdl)); + + return (1); +} + +static void +dtsec_setup_multicast(struct dtsec_softc *sc) +{ +#if (DPAA_VERSION < 11) + int i; + + if (if_getflags(sc->sc_ifnet) & IFF_ALLMULTI) { + for (i = 0; i < 8; i++) + bus_write_4(sc->sc_mem, DTSEC_REG_GADDR(i), 0xFFFFFFFF); + + return; + } + + fman_dtsec_reset_filter_table(rman_get_virtual(sc->sc_mem), + true, false); +#else + /* mEMAC: no hash reset API — SetPromiscuous(false) clears filter */ + FM_MAC_SetPromiscuous(sc->sc_mach, false); +#endif + if_foreach_llmaddr(sc->sc_ifnet, dtsec_hash_maddr, sc); +} + +static void dtsec_if_tick(void *arg); + static int dtsec_if_enable_locked(struct dtsec_softc *sc) { @@ -357,8 +437,27 @@ dtsec_if_enable_locked(struct dtsec_softc *sc) if (error != E_OK) return (EIO); + /* Reprogram hardware MAC address from current stack address. + * LAGG changes the stack MAC via if_setlladdr() then cycles + * SIOCSIFFLAGS, which reaches here — not dtsec_if_init_locked(). + * Safe to call unconditionally; no-op if address hasn't changed. */ + error = FM_MAC_ModifyMacAddr(sc->sc_mach, + (t_EnetAddr *)if_getlladdr(sc->sc_ifnet)); + if (error != E_OK) + return (EIO); + + dtsec_setup_multicast(sc); + if_setdrvflagbits(sc->sc_ifnet, IFF_DRV_RUNNING, 0); + /* Start MII tick if not already running. + * dtsec_if_init_locked() starts this, but LAGG member ports + * are enabled via SIOCSIFFLAGS → dtsec_if_enable_locked() + * without going through if_init(), so the tick never starts. + * callout_reset is idempotent — safe to call if already running. */ + if (sc->sc_mii != NULL) + callout_reset(&sc->sc_tick_callout, hz, dtsec_if_tick, sc); + /* Refresh link state */ dtsec_miibus_statchg(sc->sc_dev); @@ -389,6 +488,44 @@ dtsec_if_disable_locked(struct dtsec_softc *sc) return (0); } +static int +dtsec_if_sfp_i2c_read(struct dtsec_softc *sc, struct ifi2creq *req) +{ + struct iic_msg msgs[2]; + uint8_t reg; + int error; + + if (sc->sc_sfp_i2c == NULL || sc->sc_sfp_dev == NULL) + return (ENXIO); + if (!sc->sc_sfp_modpresent) + return (ENXIO); + if (req->len == 0 || req->len > sizeof(req->data)) + return (EINVAL); + if (req->dev_addr != 0xA0 && req->dev_addr != 0xA2) + return (EINVAL); + + reg = req->offset; + + msgs[0].slave = req->dev_addr; + msgs[0].flags = IIC_M_WR; + msgs[0].len = 1; + msgs[0].buf = ® + + msgs[1].slave = req->dev_addr; + msgs[1].flags = IIC_M_RD; + msgs[1].len = req->len; + msgs[1].buf = req->data; + + error = iicbus_request_bus(sc->sc_sfp_i2c, sc->sc_dev, IIC_INTRWAIT); + if (error != 0) + return (error); + + error = iicbus_transfer(sc->sc_sfp_i2c, msgs, 2); + iicbus_release_bus(sc->sc_sfp_i2c, sc->sc_dev); + + return (error); +} + static int dtsec_if_ioctl(if_t ifp, u_long command, caddr_t data) { @@ -423,10 +560,60 @@ dtsec_if_ioctl(if_t ifp, u_long command, caddr_t data) case SIOCGIFMEDIA: case SIOCSIFMEDIA: - error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii->mii_media, - command); + if (sc->sc_mii != NULL) + error = ifmedia_ioctl(ifp, ifr, + &sc->sc_mii->mii_media, command); + else + error = ENOTTY; break; + case SIOCGI2C: { + struct ifi2creq req; + void *uptr; + + uptr = ifr_data_get_ptr(ifr); + if (uptr == NULL) { + error = EINVAL; + break; + } + + error = copyin(uptr, &req, sizeof(req)); + if (error != 0) + break; + + error = dtsec_if_sfp_i2c_read(sc, &req); + if (error != 0) + break; + + error = copyout(&req, uptr, sizeof(req)); + break; + } + case SIOCSIFCAP: { + int mask; + + mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); + + if (mask & IFCAP_TXCSUM) { + if_togglecapenable(ifp, IFCAP_TXCSUM); + if (if_getcapenable(ifp) & IFCAP_TXCSUM) + if_sethwassist(ifp, if_gethwassist(ifp) | + (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP)); + else + if_sethwassist(ifp, if_gethwassist(ifp) & + ~(CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP)); + } + if (mask & IFCAP_TXCSUM_IPV6) { + if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); + if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6) + if_sethwassist(ifp, if_gethwassist(ifp) | + (CSUM_IP6_TCP | CSUM_IP6_UDP)); + else + if_sethwassist(ifp, if_gethwassist(ifp) & + ~(CSUM_IP6_TCP | CSUM_IP6_UDP)); + } + break; + } + default: error = ether_ioctl(ifp, command, data); } @@ -441,10 +628,11 @@ dtsec_if_tick(void *arg) sc = arg; - /* TODO */ DTSEC_LOCK(sc); - mii_tick(sc->sc_mii); + if (sc->sc_mii != NULL) + mii_tick(sc->sc_mii); + callout_reset(&sc->sc_tick_callout, hz, dtsec_if_tick, sc); DTSEC_UNLOCK(sc); @@ -472,13 +660,12 @@ dtsec_if_init_locked(struct dtsec_softc *sc) error = FM_MAC_ModifyMacAddr(sc->sc_mach, (t_EnetAddr *)if_getlladdr(sc->sc_ifnet)); if (error != E_OK) { - device_printf(sc->sc_dev, "couldn't set MAC address.\n"); + if_printf(sc->sc_ifnet, "couldn't set MAC address.\n"); goto err; } - /* Start MII polling */ - if (sc->sc_mii) - callout_reset(&sc->sc_tick_callout, hz, dtsec_if_tick, sc); + /* Start MII polling / periodic diagnostics */ + callout_reset(&sc->sc_tick_callout, hz, dtsec_if_tick, sc); if (if_getflags(sc->sc_ifnet) & IFF_UP) { error = dtsec_if_enable_locked(sc); @@ -494,7 +681,7 @@ dtsec_if_init_locked(struct dtsec_softc *sc) err: dtsec_if_deinit_locked(sc); - device_printf(sc->sc_dev, "initialization error.\n"); + if_printf(sc->sc_ifnet, "initialization error.\n"); return; } @@ -521,6 +708,12 @@ dtsec_if_start(if_t ifp) DTSEC_UNLOCK(sc); } +static void +dtsec_if_qflush(if_t ifp) +{ + /* No software queue — nothing to flush */ +} + static void dtsec_if_watchdog(if_t ifp) { @@ -529,6 +722,74 @@ dtsec_if_watchdog(if_t ifp) /** @} */ +/** + * @group SFP upstream callbacks (from sfp_fdt state machine). + * @{ + */ + +static void +dtsec_sfp_trim(char *dst, const uint8_t *src, int len) +{ + int i; + + memcpy(dst, src, len); + dst[len] = '\0'; + for (i = len - 1; i >= 0 && dst[i] == ' '; i--) + dst[i] = '\0'; +} + +static int +dtsec_sfp_module_insert(void *arg, const uint8_t *id, int id_len) +{ + struct dtsec_softc *sc = arg; + + memcpy(sc->sc_sfp_id, id, MIN((int)sizeof(sc->sc_sfp_id), id_len)); + sc->sc_sfp_modpresent = true; + + return (0); /* accept all modules */ +} + +static void +dtsec_sfp_module_remove(void *arg) +{ + struct dtsec_softc *sc = arg; + + memset(sc->sc_sfp_id, 0, sizeof(sc->sc_sfp_id)); + sc->sc_sfp_modpresent = false; + sc->sc_sfp_phy_link = false; + sc->sc_sfp_phy_speed = 0; + if_link_state_change(sc->sc_ifnet, LINK_STATE_UNKNOWN); +} + +static void +dtsec_sfp_link_up(void *arg, int speed) +{ + struct dtsec_softc *sc = arg; + + sc->sc_sfp_phy_link = true; + sc->sc_sfp_phy_speed = speed; + if_link_state_change(sc->sc_ifnet, LINK_STATE_UP); +} + +static void +dtsec_sfp_link_down(void *arg) +{ + struct dtsec_softc *sc = arg; + + sc->sc_sfp_phy_link = false; + sc->sc_sfp_phy_speed = 0; + if_link_state_change(sc->sc_ifnet, LINK_STATE_DOWN); +} + +const struct sfp_upstream_ops dtsec_sfp_ops = { + .module_insert = dtsec_sfp_module_insert, + .module_remove = dtsec_sfp_module_remove, + .link_up = dtsec_sfp_link_up, + .link_down = dtsec_sfp_link_down, +}; +/** @} */ + + /** * @group IFmedia routines. * @{ @@ -539,7 +800,8 @@ dtsec_ifmedia_upd(if_t ifp) struct dtsec_softc *sc = if_getsoftc(ifp); DTSEC_LOCK(sc); - mii_mediachg(sc->sc_mii); + if (sc->sc_mii != NULL) + mii_mediachg(sc->sc_mii); DTSEC_UNLOCK(sc); return (0); @@ -552,10 +814,35 @@ dtsec_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) DTSEC_LOCK(sc); - mii_pollstat(sc->sc_mii); - - ifmr->ifm_active = sc->sc_mii->mii_media_active; - ifmr->ifm_status = sc->sc_mii->mii_media_status; + if (sc->sc_mii != NULL) { + mii_pollstat(sc->sc_mii); + ifmr->ifm_active = sc->sc_mii->mii_media_active; + ifmr->ifm_status = sc->sc_mii->mii_media_status; + } else if (sc->sc_sfp_dev != NULL) { + /* SFP+ — link state from sfp_fdt callbacks */ + ifmr->ifm_active = IFM_ETHER | IFM_FDX; + if (sc->sc_sfp_id[SFP_CONNECTOR_OFFSET] == + SFP_CONNECTOR_RJ45) { + /* Copper module — report speed */ + switch (sc->sc_sfp_phy_speed) { + case 10000: ifmr->ifm_active |= IFM_10G_T; break; + case 5000: ifmr->ifm_active |= IFM_5000_T; break; + case 2500: ifmr->ifm_active |= IFM_2500_T; break; + case 1000: ifmr->ifm_active |= IFM_1000_T; break; + default: ifmr->ifm_active |= IFM_10G_T; break; + } + } else { + ifmr->ifm_active |= IFM_10G_SR; + } + if (sc->sc_sfp_phy_link) + ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; + else + ifmr->ifm_status = IFM_AVALID; + } else { + /* 10G port with no SFP framework — legacy always-up */ + ifmr->ifm_active = IFM_ETHER | IFM_10G_SR | IFM_FDX; + ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; + } DTSEC_UNLOCK(sc); } @@ -580,7 +867,7 @@ dtsec_configure_mode(struct dtsec_softc *sc) if (sc->sc_mode == DTSEC_MODE_REGULAR) { sc->sc_port_rx_init = dtsec_rm_fm_port_rx_init; sc->sc_port_tx_init = dtsec_rm_fm_port_tx_init; - sc->sc_start_locked = dtsec_rm_if_start_locked; + /* RM uses if_transmit, no sc_start_locked needed */ } else { sc->sc_port_rx_init = dtsec_im_fm_port_rx_init; sc->sc_port_tx_init = dtsec_im_fm_port_tx_init; @@ -591,6 +878,212 @@ dtsec_configure_mode(struct dtsec_softc *sc) (sc->sc_mode == DTSEC_MODE_REGULAR) ? "regular" : "independent"); } +static int +dtsec_sysctl_bman_free(SYSCTL_HANDLER_ARGS) +{ + struct dtsec_softc *sc = (struct dtsec_softc *)arg1; + uint32_t count; + + if (sc->sc_rx_pool == NULL) + count = 0; + else + count = bman_count(sc->sc_rx_pool); + + return (sysctl_handle_int(oidp, &count, 0, req)); +} + +static int +dtsec_sysctl_diag(SYSCTL_HANDLER_ARGS) +{ + struct dtsec_softc *sc = (struct dtsec_softc *)arg1; + int val = 0; + int error; + + error = sysctl_handle_int(oidp, &val, 0, req); + if (error || req->newptr == NULL) + return (error); + + if (val != 0) { + uint32_t bcount = sc->sc_rx_pool ? + bman_count(sc->sc_rx_pool) : 0; + printf("%s: DIAG bman_free=%u bman_total=%u bpid=%u " + "rx_fqid=%u\n", + if_name(sc->sc_ifnet), bcount, + sc->sc_rx_buf_total, sc->sc_rx_bpid, + sc->sc_rx_fqr[0] ? qman_fqr_get_base_fqid(sc->sc_rx_fqr[0]) : 0); + qman_portal_dqrr_diag(); + if (sc->sc_rxph != NULL) { + printf("%s: FMan RX: frame=%u discard=%u " + "bad=%u filter=%u oobd=%u enq=%u\n", + if_name(sc->sc_ifnet), + FM_PORT_GetCounter(sc->sc_rxph, + e_FM_PORT_COUNTERS_FRAME), + FM_PORT_GetCounter(sc->sc_rxph, + e_FM_PORT_COUNTERS_DISCARD_FRAME), + FM_PORT_GetCounter(sc->sc_rxph, + e_FM_PORT_COUNTERS_RX_BAD_FRAME), + FM_PORT_GetCounter(sc->sc_rxph, + e_FM_PORT_COUNTERS_RX_FILTER_FRAME), + FM_PORT_GetCounter(sc->sc_rxph, + e_FM_PORT_COUNTERS_RX_OUT_OF_BUFFERS_DISCARD), + FM_PORT_GetCounter(sc->sc_rxph, + e_FM_PORT_COUNTERS_ENQ_TOTAL)); + } + if (sc->sc_txph != NULL) { + printf("%s: FMan TX: frame=%u discard=%u " + "len_err=%u unsup=%u deq=%u deq_dflt=%u " + "deq_conf=%u\n", + if_name(sc->sc_ifnet), + FM_PORT_GetCounter(sc->sc_txph, + e_FM_PORT_COUNTERS_FRAME), + FM_PORT_GetCounter(sc->sc_txph, + e_FM_PORT_COUNTERS_DISCARD_FRAME), + FM_PORT_GetCounter(sc->sc_txph, + e_FM_PORT_COUNTERS_LENGTH_ERR), + FM_PORT_GetCounter(sc->sc_txph, + e_FM_PORT_COUNTERS_UNSUPPRTED_FORMAT), + FM_PORT_GetCounter(sc->sc_txph, + e_FM_PORT_COUNTERS_DEQ_TOTAL), + FM_PORT_GetCounter(sc->sc_txph, + e_FM_PORT_COUNTERS_DEQ_FROM_DEFAULT), + FM_PORT_GetCounter(sc->sc_txph, + e_FM_PORT_COUNTERS_DEQ_CONFIRM)); + } + if (sc->sc_mach != NULL) { + t_FmMacStatistics ms; + if (FM_MAC_GetStatistics(sc->sc_mach, &ms) == E_OK) { + printf("%s: MAC: rxPkts=%llu rxBytes=%llu " + "rxDiscard=%llu rxErr=%llu " + "txPkts=%llu txBytes=%llu " + "txErr=%llu\n", + if_name(sc->sc_ifnet), + (unsigned long long)ms.ifInPkts, + (unsigned long long)ms.ifInOctets, + (unsigned long long)ms.ifInDiscards, + (unsigned long long)ms.ifInErrors, + (unsigned long long)ms.ifOutPkts, + (unsigned long long)ms.ifOutOctets, + (unsigned long long)ms.ifOutErrors); + } + } + /* MEMAC register dump for link debugging */ + if (sc->sc_mem != NULL) { + uint32_t cmd_cfg = be32toh(bus_read_4(sc->sc_mem, 0x008)); + uint32_t ifmode = be32toh(bus_read_4(sc->sc_mem, 0x300)); + uint32_t ifstat = be32toh(bus_read_4(sc->sc_mem, 0x304)); + uint32_t ievent = be32toh(bus_read_4(sc->sc_mem, 0x040)); + printf("%s: MEMAC cmd_cfg=0x%08x [TX_%s RX_%s] " + "IF_MODE=0x%08x [%s] IF_STATUS=0x%08x " + "IEVENT=0x%08x\n", + if_name(sc->sc_ifnet), cmd_cfg, + (cmd_cfg & 0x01) ? "EN" : "DIS", + (cmd_cfg & 0x02) ? "EN" : "DIS", + ifmode, + (ifmode & 0x3) == 0 ? "XGMII/10G" : + (ifmode & 0x3) == 2 ? "GMII/1G" : + (ifmode & 0x3) == 4 ? "RGMII" : "unknown", + ifstat, ievent); + } + /* SFP state (managed by sfp_fdt) */ + if (sc->sc_sfp_dev != NULL) { + printf("%s: SFP: module=%s link=%s speed=%d\n", + if_name(sc->sc_ifnet), + sc->sc_sfp_modpresent ? "present" : "absent", + sc->sc_sfp_phy_link ? "up" : "down", + sc->sc_sfp_phy_speed); + /* Read DDM status byte (A2h byte 110 = 0x6E) */ + if (sc->sc_sfp_i2c != NULL && + sc->sc_sfp_modpresent) { + struct iic_msg msgs[2]; + uint8_t reg = 0x6E; + uint8_t ddm_status; + int err; + msgs[0].slave = 0x51 << 1; + msgs[0].flags = IIC_M_WR; + msgs[0].len = 1; + msgs[0].buf = ® + msgs[1].slave = 0x51 << 1; + msgs[1].flags = IIC_M_RD; + msgs[1].len = 1; + msgs[1].buf = &ddm_status; + err = iicbus_request_bus(sc->sc_sfp_i2c, + sc->sc_dev, IIC_INTRWAIT); + if (err == 0) { + err = iicbus_transfer( + sc->sc_sfp_i2c, msgs, 2); + iicbus_release_bus(sc->sc_sfp_i2c, + sc->sc_dev); + } + if (err == 0) { + printf("%s: DDM byte110=0x%02x " + "[TX_DIS_STATE=%d SOFT_TXDIS=%d " + "TX_FAULT=%d RX_LOS=%d]\n", + if_name(sc->sc_ifnet), + ddm_status, + (ddm_status >> 7) & 1, + (ddm_status >> 6) & 1, + (ddm_status >> 2) & 1, + (ddm_status >> 1) & 1); + } + } + } + } + + return (0); +} + +static int +dtsec_sysctl_sfp_info(SYSCTL_HANDLER_ARGS) +{ + struct dtsec_softc *sc = (struct dtsec_softc *)arg1; + char buf[48]; + + if (!sc->sc_sfp_modpresent) { + strlcpy(buf, "empty", sizeof(buf)); + } else { + char vendor[SFP_VENDOR_LEN + 1]; + char partnum[SFP_PARTNUM_LEN + 1]; + + dtsec_sfp_trim(vendor, &sc->sc_sfp_id[SFP_VENDOR_OFFSET], + SFP_VENDOR_LEN); + dtsec_sfp_trim(partnum, &sc->sc_sfp_id[SFP_PARTNUM_OFFSET], + SFP_PARTNUM_LEN); + snprintf(buf, sizeof(buf), "%s %s", vendor, partnum); + } + + return (sysctl_handle_string(oidp, buf, sizeof(buf), req)); +} + +static uint64_t +dtsec_get_counter(if_t ifp, ift_counter cnt) +{ + struct dtsec_softc *sc = if_getsoftc(ifp); + t_FmMacStatistics ms; + + if (sc->sc_mach != NULL && + FM_MAC_GetStatistics(sc->sc_mach, &ms) == E_OK) { + switch (cnt) { + case IFCOUNTER_IPACKETS: + return (ms.ifInPkts); + case IFCOUNTER_OPACKETS: + return (ms.ifOutPkts); + case IFCOUNTER_IBYTES: + return (ms.ifInOctets); + case IFCOUNTER_OBYTES: + return (ms.ifOutOctets); + case IFCOUNTER_IERRORS: + return (ms.ifInErrors); + case IFCOUNTER_OERRORS: + return (ms.ifOutErrors); + case IFCOUNTER_IMCASTS: + return (ms.ifInMcastPkts); + default: + break; + } + } + return (if_get_counter_default(ifp, cnt)); +} + int dtsec_attach(device_t dev) { @@ -629,6 +1122,9 @@ dtsec_attach(device_t dev) if ((error = fman_get_bushandle(parent, &sc->sc_fm_base)) != 0) return (error); + fman_get_pcd_handle(parent, &sc->sc_pcdh); + fman_get_netenv_handle(parent, &sc->sc_netenvh); + /* Configure working mode */ dtsec_configure_mode(sc); @@ -676,21 +1172,33 @@ dtsec_attach(device_t dev) return (ENXIO); } + if (sc->sc_mode == DTSEC_MODE_REGULAR) { + error = dtsec_rm_pcd_init(sc); + if (error != 0) + device_printf(dev, + "PCD init failed, continuing without RSS\n"); + } + /* Create network interface for upper layers */ ifp = sc->sc_ifnet = if_alloc(IFT_ETHER); if_setsoftc(ifp, sc); - if_setflags(ifp, IFF_SIMPLEX | IFF_BROADCAST); + if_setflags(ifp, IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST); if_setinitfn(ifp, dtsec_if_init); - if_setstartfn(ifp, dtsec_if_start); if_setioctlfn(ifp, dtsec_if_ioctl); - if_setsendqlen(ifp, IFQ_MAXLEN); + if_setgetcounterfn(ifp, dtsec_get_counter); - if (sc->sc_phy_addr >= 0) - if_initname(ifp, device_get_name(sc->sc_dev), - device_get_unit(sc->sc_dev)); - else - if_initname(ifp, "dtsec_phy", device_get_unit(sc->sc_dev)); + if (sc->sc_mode == DTSEC_MODE_REGULAR) { + /* Modern if_transmit: per-CPU TX FQs, no locks, no sendq */ + if_settransmitfn(ifp, dtsec_rm_if_transmit); + if_setqflushfn(ifp, dtsec_if_qflush); + } else { + /* Legacy if_start for Independent Mode */ + if_setstartfn(ifp, dtsec_if_start); + if_setsendqlen(ifp, IFQ_MAXLEN); + } + + if_initname(ifp, "dtsec", device_get_unit(sc->sc_dev)); /* TODO */ #if 0 @@ -698,22 +1206,68 @@ dtsec_attach(device_t dev) if_setsendqready(ifp); #endif + if_setcapabilities(ifp, IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU + | IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 + | IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); if_setcapenable(ifp, if_getcapabilities(ifp)); - - /* Attach PHY(s) */ - error = mii_attach(sc->sc_dev, &sc->sc_mii_dev, ifp, dtsec_ifmedia_upd, - dtsec_ifmedia_sts, BMSR_DEFCAPMASK, sc->sc_phy_addr, - MII_OFFSET_ANY, 0); - if (error) { - device_printf(sc->sc_dev, "attaching PHYs failed: %d\n", error); - dtsec_detach(sc->sc_dev); - return (error); + if_sethwassist(ifp, CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP | + CSUM_IP6_TCP | CSUM_IP6_UDP); + + /* Attach PHY(s) — skip for 10G ports with no external PHY + * (SFP+ uses in-band status, sc_phy_addr set to -1). */ + if (sc->sc_phy_addr >= 0) { + error = mii_attach(sc->sc_dev, &sc->sc_mii_dev, ifp, + dtsec_ifmedia_upd, dtsec_ifmedia_sts, BMSR_DEFCAPMASK, + sc->sc_phy_addr, MII_OFFSET_ANY, 0); + if (error) { + if_printf(sc->sc_ifnet, "attaching PHYs failed: " + "%d\n", error); + dtsec_detach(sc->sc_dev); + return (error); + } + sc->sc_mii = device_get_softc(sc->sc_mii_dev); } - sc->sc_mii = device_get_softc(sc->sc_mii_dev); /* Attach to stack */ ether_ifattach(ifp, sc->sc_mac_addr); + /* Set baudrate */ + if (sc->sc_eth_dev_type == ETH_10GSEC) + if_setbaudrate(ifp, IF_Gbps(10ULL)); + else + if_setbaudrate(ifp, IF_Gbps(1ULL)); + + /* 10G ports: SFP leaves default LINK_STATE_UNKNOWN (no module), + * non-SFP assumes always up (legacy) */ + if (sc->sc_phy_addr < 0 && sc->sc_sfp_dev == NULL) + if_link_state_change(ifp, LINK_STATE_UP); + + /* Add diagnostic sysctls */ + if (sc->sc_mode == DTSEC_MODE_REGULAR) { + struct sysctl_ctx_list *ctx; + struct sysctl_oid *tree; + + ctx = device_get_sysctl_ctx(dev); + tree = device_get_sysctl_tree(dev); + + SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, + "bman_free", CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, + sc, 0, dtsec_sysctl_bman_free, "IU", + "BMan free buffer count"); + SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, + "diag", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + sc, 0, dtsec_sysctl_diag, "I", + "Write 1 to dump DQRR + BMan state to dmesg"); + + if (sc->sc_sfp_dev != NULL) { + SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), + OID_AUTO, "sfp_info", + CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, + sc, 0, dtsec_sysctl_sfp_info, "A", + "SFP+ module vendor and part number"); + } + } + return (0); } @@ -740,6 +1294,9 @@ dtsec_detach(device_t dev) } if (sc->sc_mode == DTSEC_MODE_REGULAR) { + /* Free PCD (must happen before port free) */ + dtsec_rm_pcd_free(sc); + /* Free RX/TX FQRs */ dtsec_rm_fqr_rx_free(sc); dtsec_rm_fqr_tx_free(sc); @@ -820,6 +1377,9 @@ dtsec_miibus_statchg(device_t dev) DTSEC_LOCK_ASSERT(sc); + if (sc->sc_mii == NULL) + return; /* 10G port, no PHY — nothing to adjust */ + duplex = ((sc->sc_mii->mii_media_active & IFM_GMASK) == IFM_FDX); switch (IFM_SUBTYPE(sc->sc_mii->mii_media_active)) { @@ -842,6 +1402,6 @@ dtsec_miibus_statchg(device_t dev) error = FM_MAC_AdjustLink(sc->sc_mach, speed, duplex); if (error != E_OK) - device_printf(sc->sc_dev, "error while adjusting MAC speed.\n"); + if_printf(sc->sc_ifnet, "error while adjusting MAC speed.\n"); } /** @} */ diff --git a/sys/dev/dpaa/if_dtsec.h b/sys/dev/dpaa/if_dtsec.h index 4de0b776e9f5..c8f972fb109d 100644 --- a/sys/dev/dpaa/if_dtsec.h +++ b/sys/dev/dpaa/if_dtsec.h @@ -27,6 +27,10 @@ #ifndef IF_DTSEC_H_ #define IF_DTSEC_H_ +#include "opt_dpaa.h" + +#include + /** * @group dTSEC common API. * @{ @@ -34,6 +38,9 @@ #define DTSEC_MODE_REGULAR 0 #define DTSEC_MODE_INDEPENDENT 1 +#define DTSEC_TX_NUM_FQS 4 /* One per CPU (LS1046A quad-core) */ +#define DTSEC_RX_NUM_FQS 128 /* RSS hash distribution (power of 2) */ + #define DTSEC_LOCK(sc) mtx_lock(&(sc)->sc_lock) #define DTSEC_UNLOCK(sc) mtx_unlock(&(sc)->sc_lock) #define DTSEC_LOCK_ASSERT(sc) mtx_assert(&(sc)->sc_lock, MA_OWNED) @@ -65,6 +72,7 @@ struct dtsec_softc { /* dTSEC data */ enum eth_dev_type sc_eth_dev_type; uint8_t sc_eth_id; /* Ethernet ID within its frame manager */ + uint8_t sc_mac_cell_index; /* raw DT cell-index */ uintptr_t sc_mac_mem_offset; e_EnetMode sc_mac_enet_mode; int sc_mac_mdio_irq; @@ -97,20 +105,50 @@ struct dtsec_softc { uint8_t sc_rx_bpid; uma_zone_t sc_rx_zone; char sc_rx_zname[64]; + uint32_t sc_rx_buf_total; /* total buffers alloc'd from UMA */ - /* RX Frame Queue */ - t_Handle sc_rx_fqr; + /* RX Frame Queues — RSS hash distribution via dedicated channels */ + t_Handle sc_rx_fqr[DTSEC_RX_NUM_FQS]; uint32_t sc_rx_fqid; - /* TX Frame Queue */ - t_Handle sc_tx_fqr; - bool sc_tx_fqr_full; + /* Per-CPU TX Frame Queues */ + t_Handle sc_tx_fqs[DTSEC_TX_NUM_FQS]; + volatile int sc_tx_inflight[DTSEC_TX_NUM_FQS]; + + /* TX Confirmation Frame Queue (single, on pool channel) */ t_Handle sc_tx_conf_fqr; uint32_t sc_tx_conf_fqid; /* Frame Info Zone */ uma_zone_t sc_fi_zone; char sc_fi_zname[64]; + + /* TX buffer prefix data offset (from FM_PORT_GetBufferDataOffset) */ + uint32_t sc_tx_data_offset; + + /* SG table buffer zone (prefix + SG entries) */ + uma_zone_t sc_sgt_zone; + char sc_sgt_zname[64]; + uint32_t sc_sgt_buf_size; + + /* PCD (Parse-Classify-Distribute) for RSS + RX checksum */ + t_Handle sc_pcdh; + t_Handle sc_netenvh; + t_Handle sc_scheme; + uint32_t sc_rx_data_offset; + t_Handle sc_vsph; /* default VSP handle */ + + /* CEETM DSCP-based egress QoS (populated by CDX module) */ + volatile int sc_ceetm_en; + uint32_t sc_ceetm_dscp_fqid[64]; + + /* SFP module management (10G ports only, via sfp_fdt callbacks) */ + device_t sc_sfp_dev; /* sfp_fdt driver instance */ + device_t sc_sfp_i2c; /* I2C bus (for sysctl diag) */ + uint8_t sc_sfp_id[64]; /* EEPROM A0h base ID */ + bool sc_sfp_modpresent; /* module present */ + bool sc_sfp_phy_link; /* link state */ + int sc_sfp_phy_speed; /* speed in Mbps */ }; /** @} */ @@ -124,7 +162,7 @@ enum dtsec_fm_port_params { FM_PORT_LIODN_OFFSET = 0, FM_PORT_MEM_ID = 0, FM_PORT_MEM_ATTR = MEMORY_ATTR_CACHEABLE, - FM_PORT_BUFFER_SIZE = MCLBYTES, + FM_PORT_BUFFER_SIZE = (9600 + 64), /* DTSEC_MAX_FRAME_SIZE + headroom */ }; e_FmPortType dtsec_fm_port_rx_type(enum eth_dev_type type); @@ -136,6 +174,19 @@ e_FmPortType dtsec_fm_port_tx_type(enum eth_dev_type type); /** @} */ +/** + * @group dTSEC external buffer release. + * + * Used by dpaa_wifi.ko and dpaa_oh.ko to free BMan buffers from + * dtsec RX pools. Callers must also call dtsec_rm_pool_rx_refill_bpid() + * to prevent permanent pool depletion from CDX-offloaded frames that + * bypass dtsec_rm_fqr_rx_cb's normal refill path. + * @{ + */ +void dtsec_rm_buf_free_external(uint8_t bpid, void *buf); +void dtsec_rm_pool_rx_refill_bpid(uint8_t bpid); +/** @} */ + /** * @group dTSEC bus interface. * @{ @@ -151,4 +202,12 @@ int dtsec_miibus_writereg(device_t dev, int phy, int reg, void dtsec_miibus_statchg(device_t dev); /** @} */ +/** + * @group SFP upstream callbacks. + * @{ + */ +struct sfp_upstream_ops; +extern const struct sfp_upstream_ops dtsec_sfp_ops; +/** @} */ + #endif /* IF_DTSEC_H_ */ diff --git a/sys/dev/dpaa/if_dtsec_fdt.c b/sys/dev/dpaa/if_dtsec_fdt.c index f2f61777fbec..d3b52c78459b 100644 --- a/sys/dev/dpaa/if_dtsec_fdt.c +++ b/sys/dev/dpaa/if_dtsec_fdt.c @@ -35,7 +35,9 @@ #include +#ifdef __powerpc__ #include +#endif #include #include @@ -49,6 +51,9 @@ #include "miibus_if.h" +#include +#include "sff_if.h" + #include #include @@ -91,6 +96,7 @@ DRIVER_MODULE(dtsec, fman, dtsec_driver, 0, 0); DRIVER_MODULE(miibus, dtsec, miibus_driver, 0, 0); MODULE_DEPEND(dtsec, ether, 1, 1, 1); MODULE_DEPEND(dtsec, miibus, 1, 1, 1); +MODULE_DEPEND(dtsec, sff, 1, 1, 1); static int dtsec_fdt_probe(device_t dev) @@ -100,11 +106,11 @@ dtsec_fdt_probe(device_t dev) return (ENXIO); if (!ofw_bus_is_compatible(dev, "fsl,fman-dtsec") && - !ofw_bus_is_compatible(dev, "fsl,fman-xgec")) + !ofw_bus_is_compatible(dev, "fsl,fman-xgec") && + !ofw_bus_is_compatible(dev, "fsl,fman-memac")) return (ENXIO); - device_set_desc(dev, "Freescale Data Path Triple Speed Ethernet " - "Controller"); + device_set_desc(dev, "Freescale Data Path Ethernet Controller"); return (BUS_PROBE_DEFAULT); } @@ -116,7 +122,7 @@ dtsec_fdt_attach(device_t dev) device_t phy_dev; phandle_t enet_node, phy_node; phandle_t fman_rxtx_node[2]; - char phy_type[6]; + char phy_type[16]; pcell_t fman_tx_cell, mac_id; int rid; @@ -130,26 +136,47 @@ dtsec_fdt_attach(device_t dev) return (ENXIO); } - /* Get link speed */ - if (ofw_bus_is_compatible(dev, "fsl,fman-dtsec") != 0) + /* Determine MAC type / link speed */ + if (ofw_bus_is_compatible(dev, "fsl,fman-memac") != 0) { + /* mEMAC: detect 10G from phy-connection-type */ + if (OF_getprop(enet_node, "phy-connection-type", + (void *)phy_type, sizeof(phy_type)) > 0 && + (!strcmp(phy_type, "xgmii") || + !strcmp(phy_type, "10gbase-r"))) + sc->sc_eth_dev_type = ETH_10GSEC; + else + sc->sc_eth_dev_type = ETH_DTSEC; + } else if (ofw_bus_is_compatible(dev, "fsl,fman-dtsec") != 0) sc->sc_eth_dev_type = ETH_DTSEC; else if (ofw_bus_is_compatible(dev, "fsl,fman-xgec") != 0) sc->sc_eth_dev_type = ETH_10GSEC; - else + else { + device_printf(dev, "unknown MAC compatible\n"); return(ENXIO); + } /* Get PHY address */ - if (OF_getprop(enet_node, "phy-handle", (void *)&phy_node, - sizeof(phy_node)) <= 0) + if (OF_getencprop(enet_node, "phy-handle", (void *)&phy_node, + sizeof(phy_node)) <= 0) { + /* 10G ports may use managed="in-band-status" with no PHY */ + if (sc->sc_eth_dev_type == ETH_10GSEC) { + sc->sc_phy_addr = -1; + sc->sc_mdio = NULL; + goto skip_phy; + } + device_printf(dev, "missing phy-handle\n"); return (ENXIO); + } phy_node = OF_node_from_xref(phy_node); - if (OF_getprop(phy_node, "reg", (void *)&sc->sc_phy_addr, - sizeof(sc->sc_phy_addr)) <= 0) + if (OF_getencprop(phy_node, "reg", (void *)&sc->sc_phy_addr, + sizeof(sc->sc_phy_addr)) <= 0) { + device_printf(dev, "missing phy reg\n"); return (ENXIO); + } - phy_dev = OF_device_from_xref(OF_parent(phy_node)); + phy_dev = OF_device_from_xref(OF_xref_from_node(OF_parent(phy_node))); if (phy_dev == NULL) { device_printf(dev, "No PHY found.\n"); @@ -157,68 +184,131 @@ dtsec_fdt_attach(device_t dev) } sc->sc_mdio = phy_dev; +skip_phy: + + /* Register with SFP framework for 10G ports */ + if (sc->sc_eth_dev_type == ETH_10GSEC) { + phandle_t sfp_xref; + device_t sfp_dev; + + if (OF_getencprop(enet_node, "sfp", &sfp_xref, + sizeof(sfp_xref)) > 0) { + sfp_dev = OF_device_from_xref(sfp_xref); + + if (sfp_dev != NULL) { + sc->sc_sfp_dev = sfp_dev; + + /* Keep I2C bus handle for sysctl diag */ + SFF_GET_I2C_BUS(sfp_dev, &sc->sc_sfp_i2c); + + /* Register to receive module/link events */ + SFF_REGISTER_UPSTREAM(sfp_dev, + &dtsec_sfp_ops, sc); + } else { + device_printf(dev, + "SFP: sff driver not found\n"); + } + } + } /* Get MAC memory offset in SoC */ rid = 0; sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (sc->sc_mem == NULL) + if (sc->sc_mem == NULL) { + device_printf(dev, "could not alloc memory resource\n"); return (ENXIO); + } /* Get PHY connection type */ if (OF_getprop(enet_node, "phy-connection-type", (void *)phy_type, - sizeof(phy_type)) <= 0) + sizeof(phy_type)) <= 0) { + device_printf(dev, "missing phy-connection-type\n"); return (ENXIO); + } if (!strcmp(phy_type, "sgmii")) sc->sc_mac_enet_mode = e_ENET_MODE_SGMII_1000; else if (!strcmp(phy_type, "rgmii")) sc->sc_mac_enet_mode = e_ENET_MODE_RGMII_1000; else if (!strcmp(phy_type, "xgmii")) - /* We set 10 Gigabit mode flag however we don't support it */ sc->sc_mac_enet_mode = e_ENET_MODE_XGMII_10000; - else + else if (!strcmp(phy_type, "10gbase-r")) + sc->sc_mac_enet_mode = e_ENET_MODE_XGMII_10000; + else { + device_printf(dev, "unsupported phy-connection-type: %s\n", + phy_type); return (ENXIO); + } if (OF_getencprop(enet_node, "cell-index", - (void *)&mac_id, sizeof(mac_id)) <= 0) + (void *)&mac_id, sizeof(mac_id)) <= 0) { + device_printf(dev, "missing cell-index\n"); return (ENXIO); + } sc->sc_eth_id = mac_id; + sc->sc_mac_cell_index = mac_id; + /* ncsw expects 0-based per-type MAC IDs: 0..5 for 1G, 0..1 for 10G. + * FManV3 device trees use cell-index 0-5 for 1G MACs but 8-9 for + * 10G MACs (hardware MEMAC slot numbering). Remap to 0-based. */ + if (sc->sc_eth_dev_type == ETH_10GSEC && mac_id >= 8) + sc->sc_eth_id = mac_id - 8; /* Get RX/TX port handles */ - if (OF_getprop(enet_node, "fsl,fman-ports", (void *)fman_rxtx_node, - sizeof(fman_rxtx_node)) <= 0) + if (OF_getencprop(enet_node, "fsl,fman-ports", (void *)fman_rxtx_node, + sizeof(fman_rxtx_node)) <= 0) { + device_printf(dev, "missing fsl,fman-ports\n"); return (ENXIO); + } - if (fman_rxtx_node[0] == 0) + if (fman_rxtx_node[0] == 0) { + device_printf(dev, "fsl,fman-ports[0] is zero\n"); return (ENXIO); + } - if (fman_rxtx_node[1] == 0) + if (fman_rxtx_node[1] == 0) { + device_printf(dev, "fsl,fman-ports[1] is zero\n"); return (ENXIO); + } - fman_rxtx_node[0] = OF_instance_to_package(fman_rxtx_node[0]); - fman_rxtx_node[1] = OF_instance_to_package(fman_rxtx_node[1]); + /* fsl,fman-ports are phandle references on ARM64, ihandles on PPC */ + fman_rxtx_node[0] = OF_node_from_xref(fman_rxtx_node[0]); + fman_rxtx_node[1] = OF_node_from_xref(fman_rxtx_node[1]); if (ofw_bus_node_is_compatible(fman_rxtx_node[0], - "fsl,fman-v2-port-rx") == 0) + "fsl,fman-v2-port-rx") == 0 && + ofw_bus_node_is_compatible(fman_rxtx_node[0], + "fsl,fman-v3-port-rx") == 0) { + device_printf(dev, "RX port incompatible\n"); return (ENXIO); + } if (ofw_bus_node_is_compatible(fman_rxtx_node[1], - "fsl,fman-v2-port-tx") == 0) + "fsl,fman-v2-port-tx") == 0 && + ofw_bus_node_is_compatible(fman_rxtx_node[1], + "fsl,fman-v3-port-tx") == 0) { + device_printf(dev, "TX port incompatible\n"); return (ENXIO); + } /* Get RX port HW id */ - if (OF_getprop(fman_rxtx_node[0], "reg", (void *)&sc->sc_port_rx_hw_id, - sizeof(sc->sc_port_rx_hw_id)) <= 0) + if (OF_getencprop(fman_rxtx_node[0], "reg", (void *)&sc->sc_port_rx_hw_id, + sizeof(sc->sc_port_rx_hw_id)) <= 0) { + device_printf(dev, "missing RX port reg\n"); return (ENXIO); + } /* Get TX port HW id */ - if (OF_getprop(fman_rxtx_node[1], "reg", (void *)&sc->sc_port_tx_hw_id, - sizeof(sc->sc_port_tx_hw_id)) <= 0) + if (OF_getencprop(fman_rxtx_node[1], "reg", (void *)&sc->sc_port_tx_hw_id, + sizeof(sc->sc_port_tx_hw_id)) <= 0) { + device_printf(dev, "missing TX port reg\n"); return (ENXIO); + } - if (OF_getprop(fman_rxtx_node[1], "cell-index", &fman_tx_cell, - sizeof(fman_tx_cell)) <= 0) + if (OF_getencprop(fman_rxtx_node[1], "cell-index", &fman_tx_cell, + sizeof(fman_tx_cell)) <= 0) { + device_printf(dev, "missing TX port cell-index\n"); return (ENXIO); + } /* Get QMan channel */ sc->sc_port_tx_qman_chan = fman_qman_channel_id(device_get_parent(dev), fman_tx_cell); diff --git a/sys/dev/dpaa/if_dtsec_im.c b/sys/dev/dpaa/if_dtsec_im.c index fcde0df60306..0cd55dbc7c7b 100644 --- a/sys/dev/dpaa/if_dtsec_im.c +++ b/sys/dev/dpaa/if_dtsec_im.c @@ -229,7 +229,8 @@ dtsec_im_if_start_locked(struct dtsec_softc *sc) DTSEC_LOCK_ASSERT(sc); /* TODO: IFF_DRV_OACTIVE */ - if ((sc->sc_mii->mii_media_status & IFM_ACTIVE) == 0) + if (sc->sc_mii != NULL && + (sc->sc_mii->mii_media_status & IFM_ACTIVE) == 0) return; if ((if_getdrvflags(sc->sc_ifnet) & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) diff --git a/sys/dev/dpaa/if_dtsec_rm.c b/sys/dev/dpaa/if_dtsec_rm.c index b213a259b5bc..0e20f01853da 100644 --- a/sys/dev/dpaa/if_dtsec_rm.c +++ b/sys/dev/dpaa/if_dtsec_rm.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include @@ -49,13 +51,25 @@ #include "miibus_if.h" +#include "opt_dpaa.h" + +#include +#include +#include +#include +#include + #include #include #include #include +#include +#include #include +#include #include "fman.h" +#include "fman_chardev.h" #include "bman.h" #include "qman.h" #include "if_dtsec.h" @@ -69,23 +83,57 @@ #define DTSEC_BPOOLS_USED (1) #define DTSEC_MAX_TX_QUEUE_LEN 256 +#define DTSEC_RX_QUEUES 128 /* Power of 2, 7-bit hash mask like Linux */ + +/* FMan parse result L3/L4 type indicators (big-endian in parse result) */ +#define FM_L3_PARSE_RESULT_IPV4 0x8000 +#define FM_L3_PARSE_RESULT_IPV6 0x4000 +#define FM_L4_PARSE_RESULT_UDP 0x40 +#define FM_L4_PARSE_RESULT_TCP 0x20 + struct dtsec_rm_frame_info { struct mbuf *fi_mbuf; t_DpaaSGTE fi_sgt[DPAA_NUM_OF_SG_TABLE_ENTRY]; + void *fi_tx_buf; /* SG table buffer (prefix + SG entries) */ + uint8_t fi_cpu; /* CPU index for in-flight tracking */ }; enum dtsec_rm_pool_params { - DTSEC_RM_POOL_RX_LOW_MARK = 16, - DTSEC_RM_POOL_RX_HIGH_MARK = 64, - DTSEC_RM_POOL_RX_MAX_SIZE = 256, + DTSEC_RM_POOL_RX_LOW_MARK = 256, + DTSEC_RM_POOL_RX_HIGH_MARK = 2048, + DTSEC_RM_POOL_RX_MAX_SIZE = 2048, + /* + * Maximum in-flight RX buffers per port. + * + * Buffers are freed back to UMA when the mbuf is released + * (consumption model), so this bounds simultaneous allocation, + * not permanent memory commitment. The original KVA pointer + * is stashed in the buffer's privData area before handing to + * BMan, then recovered via the DMAP address after the hardware + * round-trip to pass back to uma_zfree. + * + * 4096 × 9664 = ~38 MB peak per port. + */ + DTSEC_RM_POOL_RX_MAX_TOTAL = 4096, + /* + * Active refill threshold: when free buffer count drops below + * this, the RX callback allocates new buffers. FMan acquires + * buffers directly from BMan hardware, bypassing software + * depletion callbacks — without active refill the pool exhausts + * under sustained RX load. + */ + DTSEC_RM_POOL_RX_REFILL_THRESH = 1024, + DTSEC_RM_POOL_RX_REFILL_COUNT = 256, DTSEC_RM_POOL_FI_LOW_MARK = 16, DTSEC_RM_POOL_FI_HIGH_MARK = 64, DTSEC_RM_POOL_FI_MAX_SIZE = 256, }; +#ifndef __aarch64__ #define DTSEC_RM_FQR_RX_CHANNEL e_QM_FQ_CHANNEL_POOL1 -#define DTSEC_RM_FQR_TX_CONF_CHANNEL e_QM_FQ_CHANNEL_SWPORTAL0 +#endif +#define DTSEC_RM_FQR_TX_CONF_CHANNEL e_QM_FQ_CHANNEL_POOL1 enum dtsec_rm_fqr_params { DTSEC_RM_FQR_RX_WQ = 1, DTSEC_RM_FQR_TX_WQ = 1, @@ -104,6 +152,8 @@ dtsec_rm_fi_pool_free(struct dtsec_softc *sc) if (sc->sc_fi_zone != NULL) uma_zdestroy(sc->sc_fi_zone); + if (sc->sc_sgt_zone != NULL) + uma_zdestroy(sc->sc_sgt_zone); } int @@ -178,13 +228,167 @@ dtsec_rm_fm_port_rx_init(struct dtsec_softc *sc, int unit) return (ENXIO); } + if (sc->sc_pcdh != NULL) { + /* + * Configure RX buffer prefix: parse result for checksum + * validation and hash result for RSS distribution. + * + * manipExtraSpace = 96 matches the Linux SDK DTS + * buffer-layout = <0x60 0x40> which reserves 96 bytes + * for header manipulation (NAT, encap) by CDX/FMan. + */ + t_FmBufferPrefixContent prefix; + + memset(&prefix, 0, sizeof(prefix)); + prefix.privDataSize = 16; + prefix.passPrsResult = TRUE; + prefix.passHashResult = TRUE; + prefix.dataAlign = 64; + prefix.manipExtraSpace = 96; + + error = FM_PORT_ConfigBufferPrefixContent(sc->sc_rxph, + &prefix); + if (error != E_OK) { + device_printf(sc->sc_dev, + "couldn't configure RX buffer prefix.\n"); + FM_PORT_Free(sc->sc_rxph); + sc->sc_rxph = NULL; + return (ENXIO); + } + } + +#ifdef FM_HEAVY_TRAFFIC_HANG_ERRATA_FMAN_A005669 + /* Errata A005669: discard frames with physical errors (FCS, SGMII + * disparity, FIFO overflow) in FMan hardware and increase FIFO by + * 4KB to prevent FMan hang under heavy traffic. */ + FM_PORT_ConfigBCBWorkaround(sc->sc_rxph); +#endif + error = FM_PORT_Init(sc->sc_rxph); if (error != E_OK) { device_printf(sc->sc_dev, "couldn't initialize FM Port RX.\n"); FM_PORT_Free(sc->sc_rxph); + sc->sc_rxph = NULL; return (ENXIO); } + /* Enable BMI statistics counters for PCD diagnostics */ + FM_PORT_SetStatisticsCounters(sc->sc_rxph, TRUE); + + /* + * Reserve 1 port-private policer profile slot for this RX port. + * CDX QoS creates the actual profile via FM_PCD_PlcrProfileSet() + * when the CDX module loads (before dpa_app calls FM_PORT_SetPCD). + * Must be called after FM_PORT_Init() and before FM_PORT_SetPCD(). + */ + error = FM_PORT_PcdPlcrAllocProfiles(sc->sc_rxph, 1); + if (error != E_OK) + device_printf(sc->sc_dev, + "FM_PORT_PcdPlcrAllocProfiles failed: %d\n", error); + + /* + * Allocate Virtual Storage Profiles for this RX port. + * FManv3+ (DPAA >= 11) requires per-port VSP allocation for + * the CC/PCD pipeline to work correctly after CDX/dpa_app + * replaces the default PCD. This matches the Linux SDK driver + * which reads vsp-window = <2 0> from the DTS extended-args. + * + * Must be called after FM_PORT_Init() and before FM_PORT_SetPCD() + * and FM_PORT_Enable(). + */ +#if (DPAA_VERSION >= 11) + if (sc->sc_pcdh != NULL && sc->sc_txph != NULL) { + t_FmPortVSPAllocParams vsp_alloc; + t_FmVspParams vsp_params; + t_FmBufferPrefixContent vsp_prefix; + t_Handle h_vsp; + + /* Step 1: Allocate 2 VSP slots for this port */ + memset(&vsp_alloc, 0, sizeof(vsp_alloc)); + vsp_alloc.numOfProfiles = 2; + vsp_alloc.dfltRelativeId = 0; + vsp_alloc.h_FmTxPort = sc->sc_txph; + + error = FM_PORT_VSPAlloc(sc->sc_rxph, &vsp_alloc); + if (error != E_OK) { + device_printf(sc->sc_dev, + "couldn't allocate VSP: error %d\n", error); + goto vsp_done; + } + + /* Step 2: Configure default VSP (profile 0) */ + memset(&vsp_params, 0, sizeof(vsp_params)); + vsp_params.h_Fm = sc->sc_fmh; + vsp_params.portParams.portType = + dtsec_fm_port_rx_type(sc->sc_eth_dev_type); + vsp_params.portParams.portId = sc->sc_eth_id; + vsp_params.relativeProfileId = 0; + vsp_params.liodnOffset = 0; + vsp_params.extBufPools.numOfPoolsUsed = DTSEC_BPOOLS_USED; + vsp_params.extBufPools.extBufPool[0].id = sc->sc_rx_bpid; + vsp_params.extBufPools.extBufPool[0].size = FM_PORT_BUFFER_SIZE; + + h_vsp = FM_VSP_Config(&vsp_params); + if (h_vsp == NULL) { + device_printf(sc->sc_dev, + "couldn't configure VSP.\n"); + goto vsp_done; + } + + /* Step 3: Set VSP buffer prefix (matching SDK DTS + * buffer-layout = <0x60 0x40>) */ + memset(&vsp_prefix, 0, sizeof(vsp_prefix)); + vsp_prefix.privDataSize = 16; + vsp_prefix.passPrsResult = TRUE; + vsp_prefix.passHashResult = TRUE; + vsp_prefix.dataAlign = 64; + vsp_prefix.manipExtraSpace = 96; + + error = FM_VSP_ConfigBufferPrefixContent(h_vsp, &vsp_prefix); + if (error != E_OK) { + device_printf(sc->sc_dev, + "couldn't configure VSP buffer prefix: %d\n", + error); + FM_VSP_Free(h_vsp); + goto vsp_done; + } + + /* Step 4: Initialize VSP hardware registers */ + error = FM_VSP_Init(h_vsp); + if (error != E_OK) { + device_printf(sc->sc_dev, + "couldn't initialize VSP: %d\n", error); + FM_VSP_Free(h_vsp); + goto vsp_done; + } + + sc->sc_vsph = h_vsp; + device_printf(sc->sc_dev, + "VSP allocated: 2 profiles, default ID 0\n"); +vsp_done: ; + } +#endif /* DPAA_VERSION >= 11 */ + + /* Register RX port with FMD chardev for userspace PCD control */ + { + struct fmcd_softc *fmcd; + + fmcd = fman_get_fmcd(device_get_parent(sc->sc_dev)); + if (fmcd != NULL) + fmcd_register_rx_port(fmcd, + sc->sc_mac_cell_index, sc->sc_rxph, + sc->sc_rx_fqid); + } + + if (sc->sc_pcdh != NULL) { + sc->sc_rx_data_offset = + FM_PORT_GetBufferDataOffset(sc->sc_rxph); + if (bootverbose) + device_printf(sc->sc_dev, + "RX buffer data offset: %u\n", + sc->sc_rx_data_offset); + } + if (bootverbose) device_printf(sc->sc_dev, "RX hw port 0x%02x initialized.\n", sc->sc_port_rx_hw_id); @@ -224,13 +428,57 @@ dtsec_rm_fm_port_tx_init(struct dtsec_softc *sc, int unit) return (ENXIO); } + /* Configure TX buffer prefix to include parse results area. + * When TX checksum offload is requested, the driver fills in the + * parse result and sets FM_FD_CMD_RPD | FM_FD_CMD_DTC. + * + * manipExtraSpace = 96 matches the Linux SDK DTS + * buffer-layout = <0x60 0x40> for TX port extended-args. + */ + { + t_FmBufferPrefixContent prefix; + + memset(&prefix, 0, sizeof(prefix)); + prefix.privDataSize = 16; + prefix.passPrsResult = TRUE; + prefix.passTimeStamp = FALSE; + prefix.passHashResult = FALSE; + prefix.passAllOtherPCDInfo = FALSE; + prefix.dataAlign = 64; + prefix.manipExtraSpace = 96; + + error = FM_PORT_ConfigBufferPrefixContent(sc->sc_txph, + &prefix); + if (error != E_OK) { + device_printf(sc->sc_dev, + "couldn't configure TX buffer prefix.\n"); + FM_PORT_Free(sc->sc_txph); + sc->sc_txph = NULL; + return (ENXIO); + } + } + error = FM_PORT_Init(sc->sc_txph); if (error != E_OK) { device_printf(sc->sc_dev, "couldn't initialize FM Port TX.\n"); FM_PORT_Free(sc->sc_txph); + sc->sc_txph = NULL; return (ENXIO); } + sc->sc_tx_data_offset = FM_PORT_GetBufferDataOffset(sc->sc_txph); + if (bootverbose) + device_printf(sc->sc_dev, "TX buffer data offset: %u\n", + sc->sc_tx_data_offset); + + /* Create UMA zone for SG table buffers (prefix + SG entries) */ + sc->sc_sgt_buf_size = sc->sc_tx_data_offset + + DPAA_NUM_OF_SG_TABLE_ENTRY * sizeof(t_DpaaSGTE); + snprintf(sc->sc_sgt_zname, sizeof(sc->sc_sgt_zname), "%s: SGT buf", + device_get_nameunit(sc->sc_dev)); + sc->sc_sgt_zone = uma_zcreate(sc->sc_sgt_zname, sc->sc_sgt_buf_size, + NULL, NULL, NULL, NULL, 63 /* 64-byte align */, 0); + if (bootverbose) device_printf(sc->sc_dev, "TX hw port 0x%02x initialized.\n", sc->sc_port_tx_hw_id); @@ -244,6 +492,31 @@ dtsec_rm_fm_port_tx_init(struct dtsec_softc *sc, int unit) * @group dTSEC buffer pools routines. * @{ */ + +/* + * Store the original KVA pointer from uma_zalloc in the buffer's + * private data area (first 8 bytes). FMan never touches privData + * (privDataSize=16), so this survives the BMan/FMan/QMan round-trip. + * + * After the round-trip, XX_PhysToVirt returns a DMAP address — a + * different virtual address for the same physical memory. We read + * back the original KVA pointer via the DMAP mapping to pass to + * uma_zfree, which requires the exact address it originally returned. + */ +static inline void +dtsec_rm_buf_stash_ptr(void *buf) +{ + + *(uintptr_t *)buf = (uintptr_t)buf; +} + +static inline void * +dtsec_rm_buf_recover_ptr(void *buf) +{ + + return ((void *)(*(uintptr_t *)buf)); +} + static t_Error dtsec_rm_pool_rx_put_buffer(t_Handle h_BufferPool, uint8_t *buffer, t_Handle context) @@ -251,7 +524,7 @@ dtsec_rm_pool_rx_put_buffer(t_Handle h_BufferPool, uint8_t *buffer, struct dtsec_softc *sc; sc = h_BufferPool; - uma_zfree(sc->sc_rx_zone, buffer); + uma_zfree(sc->sc_rx_zone, dtsec_rm_buf_recover_ptr(buffer)); return (E_OK); } @@ -264,27 +537,65 @@ dtsec_rm_pool_rx_get_buffer(t_Handle h_BufferPool, t_Handle *context) sc = h_BufferPool; buffer = uma_zalloc(sc->sc_rx_zone, M_NOWAIT); + if (buffer != NULL) + dtsec_rm_buf_stash_ptr(buffer); return (buffer); } +/* + * Refill the BMan buffer pool directly from UMA, bypassing + * BM_POOL_FillBufs. This avoids the NCSW RETURN_ERROR printf + * when uma_zalloc(M_NOWAIT) transiently fails — a single printf + * to serial console at 115200 baud takes ~5ms, blocking the CPU + * in interrupt context and causing cascading buffer exhaustion. + * + * Returns the number of buffers actually allocated. + */ +static unsigned int +dtsec_rm_pool_rx_refill(struct dtsec_softc *sc, unsigned int nbufs) +{ + unsigned int i; + uint8_t *buf; + + for (i = 0; i < nbufs; i++) { + buf = uma_zalloc(sc->sc_rx_zone, M_NOWAIT); + if (buf == NULL) + break; + dtsec_rm_buf_stash_ptr(buf); + if (bman_put_buffer(sc->sc_rx_pool, buf) != 0) { + uma_zfree(sc->sc_rx_zone, buf); + break; + } + } + return (i); +} + static void dtsec_rm_pool_rx_depleted(t_Handle h_App, bool in) { struct dtsec_softc *sc; - unsigned int count; + unsigned int count, headroom, nbufs; sc = h_App; if (!in) return; - while (1) { - count = bman_count(sc->sc_rx_pool); - if (count > DTSEC_RM_POOL_RX_HIGH_MARK) - return; + /* Only refill up to the per-port cap */ + if (atomic_load_32(&sc->sc_rx_buf_total) >= + DTSEC_RM_POOL_RX_MAX_TOTAL) + return; - bman_pool_fill(sc->sc_rx_pool, DTSEC_RM_POOL_RX_HIGH_MARK); + count = bman_count(sc->sc_rx_pool); + if (count < DTSEC_RM_POOL_RX_HIGH_MARK) { + headroom = DTSEC_RM_POOL_RX_MAX_TOTAL - + atomic_load_32(&sc->sc_rx_buf_total); + nbufs = DTSEC_RM_POOL_RX_HIGH_MARK - count; + if (nbufs > headroom) + nbufs = headroom; + nbufs = dtsec_rm_pool_rx_refill(sc, nbufs); + atomic_add_32(&sc->sc_rx_buf_total, nbufs); } } @@ -299,18 +610,24 @@ dtsec_rm_pool_rx_free(struct dtsec_softc *sc) uma_zdestroy(sc->sc_rx_zone); } +/* + * BPID→dtsec_softc mapping for cross-module buffer release. + * + * When dpaa_wifi.ko receives a frame from CDX (buffer from a dtsec BMan + * pool), it needs to free the buffer back to the correct dtsec's UMA zone + * and decrement sc_rx_buf_total. This table maps BPID to dtsec_softc. + */ +static struct dtsec_softc *dtsec_bpid_map[256]; + int dtsec_rm_pool_rx_init(struct dtsec_softc *sc) { - /* FM_PORT_BUFFER_SIZE must be less than PAGE_SIZE */ - CTASSERT(FM_PORT_BUFFER_SIZE < PAGE_SIZE); - snprintf(sc->sc_rx_zname, sizeof(sc->sc_rx_zname), "%s: RX Buffers", device_get_nameunit(sc->sc_dev)); sc->sc_rx_zone = uma_zcreate(sc->sc_rx_zname, FM_PORT_BUFFER_SIZE, NULL, - NULL, NULL, NULL, FM_PORT_BUFFER_SIZE - 1, 0); + NULL, NULL, NULL, 255 /* 256-byte align for DMA */, 0); sc->sc_rx_pool = bman_pool_create(&sc->sc_rx_bpid, FM_PORT_BUFFER_SIZE, 0, 0, DTSEC_RM_POOL_RX_MAX_SIZE, dtsec_rm_pool_rx_get_buffer, @@ -323,8 +640,53 @@ dtsec_rm_pool_rx_init(struct dtsec_softc *sc) return (EIO); } + sc->sc_rx_buf_total = DTSEC_RM_POOL_RX_MAX_SIZE; + + /* Register BPID→softc mapping for external buffer release */ + dtsec_bpid_map[sc->sc_rx_bpid] = sc; + return (0); } + +void +dtsec_rm_buf_free_external(uint8_t bpid, void *buf) +{ + struct dtsec_softc *sc; + + sc = dtsec_bpid_map[bpid]; + if (__predict_false(sc == NULL)) { + printf("dtsec_rm_buf_free_external: unknown BPID %u\n", bpid); + return; + } + + uma_zfree(sc->sc_rx_zone, + (void *)(*(uintptr_t *)buf)); /* recover stashed KVA */ + atomic_subtract_32(&sc->sc_rx_buf_total, 1); +} + +void +dtsec_rm_pool_rx_refill_bpid(uint8_t bpid) +{ + struct dtsec_softc *sc; + uint32_t total; + unsigned int n, added; + + sc = dtsec_bpid_map[bpid]; + if (__predict_false(sc == NULL)) + return; + + if (__predict_false(bman_count(sc->sc_rx_pool) < + DTSEC_RM_POOL_RX_REFILL_THRESH) && + atomic_load_32(&sc->sc_rx_buf_total) < + DTSEC_RM_POOL_RX_MAX_TOTAL) { + total = atomic_load_32(&sc->sc_rx_buf_total); + n = DTSEC_RM_POOL_RX_REFILL_COUNT; + if (n > DTSEC_RM_POOL_RX_MAX_TOTAL - total) + n = DTSEC_RM_POOL_RX_MAX_TOTAL - total; + added = dtsec_rm_pool_rx_refill(sc, n); + atomic_add_32(&sc->sc_rx_buf_total, added); + } +} /** @} */ @@ -335,15 +697,21 @@ dtsec_rm_pool_rx_init(struct dtsec_softc *sc) static void dtsec_rm_fqr_mext_free(struct mbuf *m) { - struct dtsec_softc *sc; void *buffer; + struct dtsec_softc *sc; buffer = m->m_ext.ext_arg1; sc = m->m_ext.ext_arg2; - if (bman_count(sc->sc_rx_pool) <= DTSEC_RM_POOL_RX_MAX_SIZE) - bman_put_buffer(sc->sc_rx_pool, buffer); - else - dtsec_rm_pool_rx_put_buffer(sc, buffer, NULL); + + /* + * Consumption model: free the buffer back to UMA using the + * original KVA pointer stashed in the privData area. The + * buffer address here is a DMAP address (from XX_PhysToVirt), + * but the stashed pointer at offset 0 is the original KVA + * address that uma_zalloc returned. + */ + uma_zfree(sc->sc_rx_zone, dtsec_rm_buf_recover_ptr(buffer)); + atomic_subtract_32(&sc->sc_rx_buf_total, 1); } static e_RxStoreResponse @@ -357,20 +725,66 @@ dtsec_rm_fqr_rx_callback(t_Handle app, t_Handle fqr, t_Handle portal, m = NULL; sc = app; - frame_va = DPAA_FD_GET_ADDR(frame); - KASSERT(DPAA_FD_GET_FORMAT(frame) == e_DPAA_FD_FORMAT_TYPE_SHORT_SBSF, - ("%s(): Got unsupported frame format 0x%02X!", __func__, - DPAA_FD_GET_FORMAT(frame))); - - KASSERT(DPAA_FD_GET_OFFSET(frame) == 0, - ("%s(): Only offset 0 is supported!", __func__)); + /* + * Refill BMan buffer pool if running low. + * FMan acquires buffers directly from BMan hardware, bypassing + * software depletion callbacks. Without active refill, the pool + * exhausts under sustained RX load: all buffers end up in mbufs + * held by the TCP stack, FMan drops every subsequent frame, and + * retransmissions are also dropped — permanent deadlock. + */ + if (__predict_false(bman_count(sc->sc_rx_pool) < + DTSEC_RM_POOL_RX_REFILL_THRESH) && + atomic_load_32(&sc->sc_rx_buf_total) < + DTSEC_RM_POOL_RX_MAX_TOTAL) { + unsigned int n, added; + uint32_t total; + + total = atomic_load_32(&sc->sc_rx_buf_total); + n = DTSEC_RM_POOL_RX_REFILL_COUNT; + if (n > DTSEC_RM_POOL_RX_MAX_TOTAL - total) + n = DTSEC_RM_POOL_RX_MAX_TOTAL - total; + added = dtsec_rm_pool_rx_refill(sc, n); + atomic_add_32(&sc->sc_rx_buf_total, added); + } - if (DPAA_FD_GET_STATUS(frame) != 0) { - device_printf(sc->sc_dev, "RX error: 0x%08X\n", - DPAA_FD_GET_STATUS(frame)); + frame_va = DPAA_FD_GET_ADDR(frame); + if (frame_va == NULL) + return (e_RX_STORE_RESPONSE_CONTINUE); + + if (DPAA_FD_GET_FORMAT(frame) != e_DPAA_FD_FORMAT_TYPE_SHORT_SBSF) { + /* SG/compound FDs should not occur with our buffer pool size. + * Log and drop rather than panic. */ + if_printf(sc->sc_ifnet, + "RX: unexpected FD format 0x%02X, dropping\n", + DPAA_FD_GET_FORMAT(frame)); goto err; } + { + uint32_t fd_st = DPAA_FD_GET_STATUS(frame); + uint32_t err_bits = fd_st & + (FM_FD_RX_STATUS_ERR_MASK & ~FM_FD_ERR_NO_SCHEME); + if (err_bits) { + static volatile uint32_t rx_err_cnt; + uint32_t n = atomic_fetchadd_32(&rx_err_cnt, 1) + 1; + if (n <= 10 || (n % 100) == 0) + if_printf(sc->sc_ifnet, + "RX error #%u: fd_status=0x%08X" + " (phy=%d cls=%d red=%d yel=%d" + " plcr=%d prs=%d ext=%d)\n", + n, fd_st, + !!(fd_st & FM_FD_ERR_PHYSICAL), + !!(fd_st & FM_FD_ERR_CLS_DISCARD), + !!(fd_st & FM_FD_ERR_COLOR_RED), + !!(fd_st & FM_FD_ERR_COLOR_YELLOW), + !!(fd_st & FM_FD_ERR_ILL_PLCR), + !!(fd_st & FM_FD_ERR_PRS_HDR_ERR), + !!(fd_st & FM_FD_ERR_EXTRACTION)); + goto err; + } + } + m = m_gethdr(M_NOWAIT, MT_HEADER); if (m == NULL) goto err; @@ -380,15 +794,69 @@ dtsec_rm_fqr_rx_callback(t_Handle app, t_Handle fqr, t_Handle portal, EXT_NET_DRV); m->m_pkthdr.rcvif = sc->sc_ifnet; + m->m_data = (char *)frame_va + DPAA_FD_GET_OFFSET(frame); m->m_len = DPAA_FD_GET_LENGTH(frame); - m_fixhdr(m); + m->m_pkthdr.len = m->m_len; + + /* Extract RX checksum result from parse result in buffer prefix */ + if (sc->sc_rx_data_offset > 0) { + t_FmPrsResult *prs; + + prs = FM_PORT_GetBufferPrsResult(sc->sc_rxph, + (char *)frame_va); + if (prs != NULL) { + uint16_t l3r = be16toh(prs->l3r); + + if (l3r & (FM_L3_PARSE_RESULT_IPV4 | + FM_L3_PARSE_RESULT_IPV6)) { + m->m_pkthdr.csum_flags |= + CSUM_L3_CALC | CSUM_L3_VALID; + + if (prs->l4r & (FM_L4_PARSE_RESULT_TCP | + FM_L4_PARSE_RESULT_UDP)) { + uint32_t fd_status = + DPAA_FD_GET_STATUS(frame); + /* + * DCL4C = "Didn't Compute L4 Checksum". + * Only claim HW-verified when clear. + * When set, leave flags alone so the + * stack verifies L4 csum in software. + * + * csum_data = 0xffff: tells the stack + * the full checksum (incl pseudo-hdr) + * is valid. Without this, the stack + * reads csum_data=0 and drops the + * packet as a checksum failure. + */ + if (!(fd_status & FM_FD_CMD_DCL4C)) { + m->m_pkthdr.csum_flags |= + CSUM_L4_CALC | + CSUM_L4_VALID; + m->m_pkthdr.csum_data = + 0xffff; + } + } + } + } + } + if_inc_counter(sc->sc_ifnet, IFCOUNTER_IPACKETS, 1); + if_inc_counter(sc->sc_ifnet, IFCOUNTER_IBYTES, m->m_pkthdr.len); +#ifdef __aarch64__ + qman_rx_defer(m); +#else if_input(sc->sc_ifnet, m); +#endif return (e_RX_STORE_RESPONSE_CONTINUE); err: - bman_put_buffer(sc->sc_rx_pool, frame_va); + if_inc_counter(sc->sc_ifnet, IFCOUNTER_IERRORS, 1); + if (frame_va != NULL) { + uma_zfree(sc->sc_rx_zone, + dtsec_rm_buf_recover_ptr(frame_va)); + atomic_subtract_32(&sc->sc_rx_buf_total, 1); + } if (m != NULL) m_freem(m); @@ -401,49 +869,93 @@ dtsec_rm_fqr_tx_confirm_callback(t_Handle app, t_Handle fqr, t_Handle portal, { struct dtsec_rm_frame_info *fi; struct dtsec_softc *sc; - unsigned int qlen; t_DpaaSGTE *sgt0; + void *buf; + int cpu; sc = app; - if (DPAA_FD_GET_STATUS(frame) != 0) - device_printf(sc->sc_dev, "TX error: 0x%08X\n", - DPAA_FD_GET_STATUS(frame)); - - /* - * We are storing struct dtsec_rm_frame_info in first entry - * of scatter-gather table. - */ - sgt0 = DPAA_FD_GET_ADDR(frame); - fi = DPAA_SGTE_GET_ADDR(sgt0); + buf = DPAA_FD_GET_ADDR(frame); + if (buf == NULL) + return (e_RX_STORE_RESPONSE_CONTINUE); + + if (DPAA_FD_GET_FORMAT(frame) == e_DPAA_FD_FORMAT_TYPE_SHORT_MBSF && + DPAA_FD_GET_OFFSET(frame) > 0) { + /* SG+prefix FD (checksum offload path). + * fi pointer stored in private data area at start of buf. + * SG table buffer allocated from sc_sgt_zone. */ + fi = *(struct dtsec_rm_frame_info **)buf; + cpu = fi->fi_cpu; + m_freem(fi->fi_mbuf); + uma_zfree(sc->sc_sgt_zone, buf); + dtsec_rm_fi_free(sc, fi); + } else { + /* Legacy SG FD (no prefix, offset=0). + * fi pointer stored in first SG entry. */ + sgt0 = buf; + fi = DPAA_SGTE_GET_ADDR(sgt0); + cpu = fi->fi_cpu; + m_freem(fi->fi_mbuf); + dtsec_rm_fi_free(sc, fi); + } - /* Free transmitted frame */ - m_freem(fi->fi_mbuf); - dtsec_rm_fi_free(sc, fi); + atomic_subtract_int(&sc->sc_tx_inflight[cpu], 1); - qlen = qman_fqr_get_counter(sc->sc_tx_conf_fqr, 0, - e_QM_FQR_COUNTERS_FRAME); + return (e_RX_STORE_RESPONSE_CONTINUE); +} - if (qlen == 0) { - DTSEC_LOCK(sc); +void +dtsec_rm_fqr_rx_free(struct dtsec_softc *sc) +{ + int i; - if (sc->sc_tx_fqr_full) { - sc->sc_tx_fqr_full = 0; - dtsec_rm_if_start_locked(sc); + for (i = 0; i < DTSEC_RX_QUEUES; i++) { + if (sc->sc_rx_fqr[i] != NULL) { + qman_fqr_free(sc->sc_rx_fqr[i]); + sc->sc_rx_fqr[i] = NULL; } - - DTSEC_UNLOCK(sc); } - - return (e_RX_STORE_RESPONSE_CONTINUE); } +/* + * dtsec_release_rss_fqrs — Free RSS FQRs 1..127 from all dtsec ports. + * + * Called by CDX module during takeover, before creating CDX distribution + * FQRs. After CDX replaces the driver's PCD, the RSS FQRs are orphaned + * (no KeyGen scheme directs traffic to them). FQR 0 (base FQID) is kept + * alive — CDX miss-action routes unmatched frames there for stack delivery. + * + * Safe to call at boot before any traffic has flowed (instant, no drain). + */ void -dtsec_rm_fqr_rx_free(struct dtsec_softc *sc) +dtsec_release_rss_fqrs(void) { + devclass_t dc; + device_t dev; + struct dtsec_softc *sc; + int i, unit; - if (sc->sc_rx_fqr) - qman_fqr_free(sc->sc_rx_fqr); + dc = devclass_find("dtsec"); + if (dc == NULL) + return; + + for (unit = 0; unit < devclass_get_maxunit(dc); unit++) { + dev = devclass_get_device(dc, unit); + if (dev == NULL) + continue; + sc = device_get_softc(dev); + if (sc == NULL || sc->sc_rx_fqr[1] == NULL) + continue; + for (i = 1; i < DTSEC_RX_QUEUES; i++) { + if (sc->sc_rx_fqr[i] != NULL) { + qman_fqr_free(sc->sc_rx_fqr[i]); + sc->sc_rx_fqr[i] = NULL; + } + } + device_printf(dev, + "released %d RSS FQRs for CDX (base FQID %u kept)\n", + DTSEC_RX_QUEUES - 1, sc->sc_rx_fqid); + } } int @@ -451,17 +963,103 @@ dtsec_rm_fqr_rx_init(struct dtsec_softc *sc) { t_Error error; t_Handle fqr; + uint32_t base_fqid; + int i; + +#ifdef __aarch64__ + /* + * RSS via per-CPU dedicated QMan channels (matching Linux). + * + * Create DTSEC_RX_QUEUES (128) single-FQID FQRs with contiguous + * FQIDs, round-robin assigned to dedicated SW portal channels: + * FQID base+0 -> SWPORTAL0, base+1 -> SWPORTAL1, ... + * base+4 -> SWPORTAL0, base+5 -> SWPORTAL1, etc. + * + * KeyGen hashes 4-tuple -> baseFqid + (hash & 127). With a + * 7-bit hash mask, different flows distribute well across CPUs. + * Same flow always hashes to same FQID -> same CPU (no reorder). + * + * The first FQR is allocated with alignment = DTSEC_RX_QUEUES + * to satisfy the KeyGen alignment requirement. The remaining + * 127 FQRs use force_fqid for contiguous FQIDs. + */ - /* Default Frame Queue */ - fqr = qman_fqr_create(1, DTSEC_RM_FQR_RX_CHANNEL, DTSEC_RM_FQR_RX_WQ, - false, 0, false, false, true, false, 0, 0, 0); + /* FQID 0: allocate with alignment to get contiguous base */ + fqr = qman_fqr_create(1, + (e_QmFQChannel)(e_QM_FQ_CHANNEL_SWPORTAL0), + DTSEC_RM_FQR_RX_WQ, false, + DTSEC_RX_QUEUES, /* alignment */ + false, false, true, false, 0, 0, 0); if (fqr == NULL) { - device_printf(sc->sc_dev, "could not create default RX queue" - "\n"); + device_printf(sc->sc_dev, + "could not create aligned RX FQR for FQID 0\n"); return (EIO); } - sc->sc_rx_fqr = fqr; + sc->sc_rx_fqr[0] = fqr; + base_fqid = qman_fqr_get_base_fqid(fqr); + sc->sc_rx_fqid = base_fqid; + + error = qman_fqr_register_cb(fqr, dtsec_rm_fqr_rx_callback, sc); + if (error != E_OK) { + device_printf(sc->sc_dev, + "could not register RX callback for FQID 0\n"); + qman_fqr_free(fqr); + sc->sc_rx_fqr[0] = NULL; + return (EIO); + } + + /* FQIDs 1..N-1: force-allocate, round-robin across CPUs */ + for (i = 1; i < DTSEC_RX_QUEUES; i++) { + e_QmFQChannel channel = + (e_QmFQChannel)(e_QM_FQ_CHANNEL_SWPORTAL0 + + (i % DTSEC_TX_NUM_FQS)); + + fqr = qman_fqr_create(1, channel, DTSEC_RM_FQR_RX_WQ, + true, base_fqid + i, + false, false, true, false, 0, 0, 0); + if (fqr == NULL) { + device_printf(sc->sc_dev, + "could not create RX FQR %d " + "(FQID %u, channel %d)\n", + i, base_fqid + i, (int)channel); + goto fail; + } + + sc->sc_rx_fqr[i] = fqr; + + error = qman_fqr_register_cb(fqr, + dtsec_rm_fqr_rx_callback, sc); + if (error != E_OK) { + device_printf(sc->sc_dev, + "could not register RX callback for FQ %d\n", i); + qman_fqr_free(fqr); + sc->sc_rx_fqr[i] = NULL; + goto fail; + } + } + + device_printf(sc->sc_dev, + "RX RSS: base FQID %u, %d queues across %d CPUs\n", + base_fqid, DTSEC_RX_QUEUES, DTSEC_TX_NUM_FQS); + + return (0); + +fail: + dtsec_rm_fqr_rx_free(sc); + return (EIO); + +#else /* !__aarch64__ — PowerPC: single FQR on pool channel */ + fqr = qman_fqr_create(DTSEC_RX_QUEUES, DTSEC_RM_FQR_RX_CHANNEL, + DTSEC_RM_FQR_RX_WQ, false, + DTSEC_RX_QUEUES > 1 ? DTSEC_RX_QUEUES : 0, + false, false, true, false, 0, 0, 0); + if (fqr == NULL) { + device_printf(sc->sc_dev, "could not create RX queue range\n"); + return (EIO); + } + + sc->sc_rx_fqr[0] = fqr; sc->sc_rx_fqid = qman_fqr_get_base_fqid(fqr); error = qman_fqr_register_cb(fqr, dtsec_rm_fqr_rx_callback, sc); @@ -471,18 +1069,30 @@ dtsec_rm_fqr_rx_init(struct dtsec_softc *sc) return (EIO); } + if (bootverbose) + device_printf(sc->sc_dev, "RX FQR: base FQID %u, %d queues\n", + sc->sc_rx_fqid, DTSEC_RX_QUEUES); + return (0); +#endif } void dtsec_rm_fqr_tx_free(struct dtsec_softc *sc) { + int i; - if (sc->sc_tx_fqr) - qman_fqr_free(sc->sc_tx_fqr); + for (i = 0; i < DTSEC_TX_NUM_FQS; i++) { + if (sc->sc_tx_fqs[i] != NULL) { + qman_fqr_free(sc->sc_tx_fqs[i]); + sc->sc_tx_fqs[i] = NULL; + } + } - if (sc->sc_tx_conf_fqr) + if (sc->sc_tx_conf_fqr != NULL) { qman_fqr_free(sc->sc_tx_conf_fqr); + sc->sc_tx_conf_fqr = NULL; + } } int @@ -490,19 +1100,24 @@ dtsec_rm_fqr_tx_init(struct dtsec_softc *sc) { t_Error error; t_Handle fqr; - - /* TX Frame Queue */ - fqr = qman_fqr_create(1, sc->sc_port_tx_qman_chan, - DTSEC_RM_FQR_TX_WQ, false, 0, false, false, true, false, 0, 0, 0); - if (fqr == NULL) { - device_printf(sc->sc_dev, "could not create default TX queue" - "\n"); - return (EIO); + int i; + + /* Per-CPU TX Frame Queues — all target the same FMan TX port channel */ + for (i = 0; i < DTSEC_TX_NUM_FQS; i++) { + fqr = qman_fqr_create(1, sc->sc_port_tx_qman_chan, + DTSEC_RM_FQR_TX_WQ, false, 0, false, false, true, false, + 0, 0, 0); + if (fqr == NULL) { + device_printf(sc->sc_dev, + "could not create TX queue for CPU %d\n", i); + dtsec_rm_fqr_tx_free(sc); + return (EIO); + } + sc->sc_tx_fqs[i] = fqr; + sc->sc_tx_inflight[i] = 0; } - sc->sc_tx_fqr = fqr; - - /* TX Confirmation Frame Queue */ + /* TX Confirmation Frame Queue (single, on pool channel) */ fqr = qman_fqr_create(1, DTSEC_RM_FQR_TX_CONF_CHANNEL, DTSEC_RM_FQR_TX_CONF_WQ, false, 0, false, false, true, false, 0, 0, 0); @@ -524,129 +1139,613 @@ dtsec_rm_fqr_tx_init(struct dtsec_softc *sc) return (EIO); } + if (bootverbose) + device_printf(sc->sc_dev, + "TX FQs: %d per-CPU queues, confirm FQID %u\n", + DTSEC_TX_NUM_FQS, sc->sc_tx_conf_fqid); + return (0); } /** @} */ /** - * @group dTSEC IFnet routines. + * @group dTSEC PCD (Parse-Classify-Distribute) routines. * @{ */ + +/** + * Initialize PCD on the RX port: create a KeyGen scheme for RSS + * hash distribution across DTSEC_RX_QUEUES, then attach PCD + * (Parser + KeyGen) to the RX port. + * + * Called from dtsec_attach() after RX port and FQR are initialized. + * Non-fatal: returns 0 on success or if PCD is unavailable. + */ +int +dtsec_rm_pcd_init(struct dtsec_softc *sc) +{ + t_FmPcdKgSchemeParams scheme; + t_FmPortPcdParams port_pcd; + t_FmPortPcdPrsParams prs; + t_FmPortPcdKgParams kg; + t_Error error; + int i; + + if (sc->sc_pcdh == NULL) + return (0); + + /* --- Create KeyGen scheme --- */ + memset(&scheme, 0, sizeof(scheme)); + scheme.modify = FALSE; + scheme.id.relativeSchemeId = device_get_unit(sc->sc_dev); + scheme.alwaysDirect = TRUE; + scheme.useHash = TRUE; + scheme.bypassFqidGeneration = FALSE; + scheme.baseFqid = sc->sc_rx_fqid; + scheme.nextEngine = e_FM_PCD_DONE; + scheme.kgNextEngineParams.doneAction = e_FM_PCD_ENQ_FRAME; + + /* + * Extract 4-tuple for hash: IPv4 SRC/DST + L4 SRC/DST port. + * Using HEADER_TYPE_TCP for L4 ports — the KeyGen's known-field + * registers (KG_SCH_KN_L4PSRC/L4PDST) are protocol-agnostic and + * extract L4 ports for both TCP and UDP. For non-IP/non-L4 + * frames, the default value (zeros) is used, so they all hash + * to baseFqid + 0. + */ + i = 0; + + /* IPv4 source IP */ + scheme.keyExtractAndHashParams.extractArray[i].type = + e_FM_PCD_EXTRACT_BY_HDR; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.hdr = + HEADER_TYPE_IPv4; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.hdrIndex = + e_FM_PCD_HDR_INDEX_NONE; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr + .ignoreProtocolValidation = FALSE; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.type = + e_FM_PCD_EXTRACT_FULL_FIELD; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr + .extractByHdrType.fullField.ipv4 = NET_HEADER_FIELD_IPv4_SRC_IP; + i++; + + /* IPv4 destination IP */ + scheme.keyExtractAndHashParams.extractArray[i].type = + e_FM_PCD_EXTRACT_BY_HDR; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.hdr = + HEADER_TYPE_IPv4; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.hdrIndex = + e_FM_PCD_HDR_INDEX_NONE; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr + .ignoreProtocolValidation = FALSE; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.type = + e_FM_PCD_EXTRACT_FULL_FIELD; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr + .extractByHdrType.fullField.ipv4 = NET_HEADER_FIELD_IPv4_DST_IP; + i++; + + /* TCP/UDP source port (KG_SCH_KN_L4PSRC covers both TCP and UDP) */ + scheme.keyExtractAndHashParams.extractArray[i].type = + e_FM_PCD_EXTRACT_BY_HDR; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.hdr = + HEADER_TYPE_TCP; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.hdrIndex = + e_FM_PCD_HDR_INDEX_NONE; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr + .ignoreProtocolValidation = FALSE; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.type = + e_FM_PCD_EXTRACT_FULL_FIELD; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr + .extractByHdrType.fullField.tcp = NET_HEADER_FIELD_TCP_PORT_SRC; + i++; + + /* TCP/UDP destination port (KG_SCH_KN_L4PDST covers both TCP and UDP) */ + scheme.keyExtractAndHashParams.extractArray[i].type = + e_FM_PCD_EXTRACT_BY_HDR; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.hdr = + HEADER_TYPE_TCP; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.hdrIndex = + e_FM_PCD_HDR_INDEX_NONE; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr + .ignoreProtocolValidation = FALSE; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr.type = + e_FM_PCD_EXTRACT_FULL_FIELD; + scheme.keyExtractAndHashParams.extractArray[i].extractByHdr + .extractByHdrType.fullField.tcp = NET_HEADER_FIELD_TCP_PORT_DST; + i++; + + scheme.keyExtractAndHashParams.numOfUsedExtracts = i; + scheme.keyExtractAndHashParams.hashDistributionNumOfFqids = + DTSEC_RX_QUEUES; + scheme.keyExtractAndHashParams.hashShift = 0; + scheme.keyExtractAndHashParams.hashDistributionFqidsShift = 0; + scheme.keyExtractAndHashParams.symmetricHash = FALSE; + + /* Defaults for missing headers: use privateDflt0 (all zeros) */ + scheme.keyExtractAndHashParams.numOfUsedDflts = 2; + scheme.keyExtractAndHashParams.dflts[0].type = e_FM_PCD_KG_IP_ADDR; + scheme.keyExtractAndHashParams.dflts[0].dfltSelect = + e_FM_PCD_KG_DFLT_PRIVATE_0; + scheme.keyExtractAndHashParams.dflts[1].type = e_FM_PCD_KG_L4_PORT; + scheme.keyExtractAndHashParams.dflts[1].dfltSelect = + e_FM_PCD_KG_DFLT_PRIVATE_0; + + sc->sc_scheme = FM_PCD_KgSchemeSet(sc->sc_pcdh, &scheme); + if (sc->sc_scheme == NULL) { + device_printf(sc->sc_dev, "couldn't create KeyGen scheme\n"); + return (ENXIO); + } + + device_printf(sc->sc_dev, + "KeyGen scheme created: baseFqid=%u, queues=%d\n", + sc->sc_rx_fqid, DTSEC_RX_QUEUES); + + /* --- Attach PCD to RX port --- */ + memset(&port_pcd, 0, sizeof(port_pcd)); + port_pcd.pcdSupport = e_FM_PORT_PCD_SUPPORT_PRS_AND_KG; + port_pcd.h_NetEnv = sc->sc_netenvh; + + memset(&prs, 0, sizeof(prs)); + prs.firstPrsHdr = HEADER_TYPE_ETH; + port_pcd.p_PrsParams = &prs; + + memset(&kg, 0, sizeof(kg)); + kg.numOfSchemes = 1; + kg.h_Schemes[0] = sc->sc_scheme; + kg.directScheme = TRUE; + kg.h_DirectScheme = sc->sc_scheme; + port_pcd.p_KgParams = &kg; + + error = FM_PORT_SetPCD(sc->sc_rxph, &port_pcd); + if (error != E_OK) { + device_printf(sc->sc_dev, + "couldn't set PCD on RX port: %d\n", error); + FM_PCD_KgSchemeDelete(sc->sc_scheme); + sc->sc_scheme = NULL; + return (ENXIO); + } + + device_printf(sc->sc_dev, "PCD enabled: PRS+KG on RX port\n"); + + + /* Register scheme with chardev so it can tear down PCD when + * userspace (FMC/dpa_app) takes over PCD management. */ + { + struct fmcd_softc *fmcd; + + fmcd = fman_get_fmcd(device_get_parent(sc->sc_dev)); + if (fmcd != NULL) + fmcd_register_rx_port_scheme(fmcd, + sc->sc_mac_cell_index, + sc->sc_scheme, &sc->sc_scheme); + } + + return (0); +} + void -dtsec_rm_if_start_locked(struct dtsec_softc *sc) +dtsec_rm_pcd_free(struct dtsec_softc *sc) { - vm_size_t dsize, psize, ssize; + + if (sc->sc_scheme != NULL) { + FM_PORT_DeletePCD(sc->sc_rxph); + FM_PCD_KgSchemeDelete(sc->sc_scheme); + sc->sc_scheme = NULL; + } +} +/** @} */ + + +/** + * Fill parse result for TX checksum offload. + * Returns 0 on success, non-zero if the frame can't be checksummed by HW. + */ +static int +dtsec_rm_tx_csum_fill(struct dtsec_softc *sc, struct mbuf *m, + t_FmPrsResult *prs) +{ + struct ether_header *eh; + struct ip *ip; + struct ip6_hdr *ip6; + uint16_t etype; + int ehlen; + + if (m->m_len < sizeof(*eh)) + return (EINVAL); + + eh = mtod(m, struct ether_header *); + etype = ntohs(eh->ether_type); + ehlen = sizeof(*eh); + + /* Handle VLAN-tagged frames */ + if (etype == ETHERTYPE_VLAN) { + if (m->m_len < sizeof(*eh) + 4) + return (EINVAL); + etype = ntohs(*(uint16_t *)((char *)eh + sizeof(*eh) + 2)); + ehlen += 4; + } + + memset(prs, 0, sizeof(*prs)); + + switch (etype) { + case ETHERTYPE_IP: + if (m->m_len < ehlen + sizeof(*ip)) + return (EINVAL); + prs->l3r = htobe16(FM_L3_PARSE_RESULT_IPV4); + ip = (struct ip *)((char *)eh + ehlen); + prs->ip_off[0] = ehlen; + prs->l4_off = ehlen + (ip->ip_hl << 2); + switch (ip->ip_p) { + case IPPROTO_TCP: + prs->l4r = FM_L4_PARSE_RESULT_TCP; + break; + case IPPROTO_UDP: + prs->l4r = FM_L4_PARSE_RESULT_UDP; + break; + default: + return (EINVAL); + } + break; + + case ETHERTYPE_IPV6: + if (m->m_len < ehlen + sizeof(*ip6)) + return (EINVAL); + prs->l3r = htobe16(FM_L3_PARSE_RESULT_IPV6); + ip6 = (struct ip6_hdr *)((char *)eh + ehlen); + prs->ip_off[0] = ehlen; + prs->l4_off = ehlen + sizeof(*ip6); + switch (ip6->ip6_nxt) { + case IPPROTO_TCP: + prs->l4r = FM_L4_PARSE_RESULT_TCP; + break; + case IPPROTO_UDP: + prs->l4r = FM_L4_PARSE_RESULT_UDP; + break; + default: + return (EINVAL); + } + break; + + default: + return (EINVAL); + } + + return (0); +} + +/** + * @group dTSEC IFnet routines. + * @{ + */ + +/** + * Extract DSCP from the IP header of an Ethernet frame. + * Returns 0 for non-IP or unparseable frames (DSCP 0 maps to + * unmapped in the CEETM array, falling back to the default FQ). + */ +static inline uint8_t +dtsec_rm_extract_dscp(struct mbuf *m) +{ + struct ether_header *eh; + uint16_t etype; + int ehlen; + + if (__predict_false(m->m_len < (int)sizeof(*eh))) + return (0); + + eh = mtod(m, struct ether_header *); + etype = ntohs(eh->ether_type); + ehlen = sizeof(*eh); + + if (etype == ETHERTYPE_VLAN) { + if (__predict_false(m->m_len < ehlen + 4)) + return (0); + etype = ntohs(*(uint16_t *)((char *)eh + ehlen + 2)); + ehlen += 4; + } + + if (etype == ETHERTYPE_IP) { + struct ip *ip; + if (__predict_false(m->m_len < ehlen + (int)sizeof(*ip))) + return (0); + ip = (struct ip *)((char *)eh + ehlen); + return ((ip->ip_tos >> 2) & 0x3f); + } + + if (etype == ETHERTYPE_IPV6) { + struct ip6_hdr *ip6; + if (__predict_false(m->m_len < ehlen + (int)sizeof(*ip6))) + return (0); + ip6 = (struct ip6_hdr *)((char *)eh + ehlen); + return ((ntohl(ip6->ip6_flow) >> 22) & 0x3f); + } + + return (0); +} + +/** + * Modern if_transmit entry point for RM (Regular Mode). + * + * Each CPU builds the FD and enqueues to its own per-CPU TX FQ on its + * own QMan portal. No locks, no sendq, no contention. The stack + * calls this directly with one mbuf at a time. + */ +int +dtsec_rm_if_transmit(if_t ifp, struct mbuf *m0) +{ + struct dtsec_softc *sc; struct dtsec_rm_frame_info *fi; - unsigned int qlen, i; - struct mbuf *m0, *m; + vm_size_t dsize, psize, ssize; + unsigned int i; + struct mbuf *m; vm_offset_t vaddr; t_DpaaFD fd; + int cpu, error; + t_DpaaSGTE *sgt; + void *sgt_buf; - DTSEC_LOCK_ASSERT(sc); - /* TODO: IFF_DRV_OACTIVE */ + sc = if_getsoftc(ifp); - if ((sc->sc_mii->mii_media_status & IFM_ACTIVE) == 0) - return; + if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))) { + m_freem(m0); + return (ENETDOWN); + } - if ((if_getdrvflags(sc->sc_ifnet) & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) - return; + /* Pin to current CPU for consistent FQ selection and portal use */ + sched_pin(); + cpu = curcpu % DTSEC_TX_NUM_FQS; - while (!if_sendq_empty(sc->sc_ifnet)) { - /* Check length of the TX queue */ - qlen = qman_fqr_get_counter(sc->sc_tx_fqr, 0, - e_QM_FQR_COUNTERS_FRAME); + /* CEETM DSCP-based FQ selection for software-path packets */ + uint32_t ceetm_fqid = 0; + if (__predict_false(sc->sc_ceetm_en)) { + uint8_t dscp = dtsec_rm_extract_dscp(m0); + ceetm_fqid = sc->sc_ceetm_dscp_fqid[dscp]; + } - if (qlen >= DTSEC_MAX_TX_QUEUE_LEN) { - sc->sc_tx_fqr_full = 1; - return; - } + /* Check per-CPU in-flight limit (software counter, no portal read) */ + if (__predict_false(atomic_load_int(&sc->sc_tx_inflight[cpu]) >= + DTSEC_MAX_TX_QUEUE_LEN)) { + sched_unpin(); + m_freem(m0); + if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); + return (ENOBUFS); + } - fi = dtsec_rm_fi_alloc(sc); - if (fi == NULL) - return; + fi = dtsec_rm_fi_alloc(sc); + if (__predict_false(fi == NULL)) { + sched_unpin(); + m_freem(m0); + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + return (ENOBUFS); + } + + fi->fi_cpu = cpu; + + fi->fi_tx_buf = NULL; + + /* + * TX checksum offload: build SG FD with buffer prefix + * containing parse result. SG entries point directly to + * mbuf fragment data — zero-copy, no linearization. + */ + if ((m0->m_pkthdr.csum_flags & + (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP | + CSUM_IP6_TCP | CSUM_IP6_UDP)) && + sc->sc_tx_data_offset > 0) { + t_FmPrsResult prs; + t_FmPrsResult *buf_prs; + + /* Fill parse result for hardware checksum */ + if (dtsec_rm_tx_csum_fill(sc, m0, &prs) != 0) { + /* Can't offload L4 (e.g. ICMP) — compute + * IP header checksum in software since the + * stack left ip_sum=0 relying on CSUM_IP. */ + if (m0->m_pkthdr.csum_flags & CSUM_IP) { + struct ether_header *eh2; + struct ip *ip2; + uint16_t et; + int ehl; + + eh2 = mtod(m0, struct ether_header *); + et = ntohs(eh2->ether_type); + ehl = sizeof(*eh2); + if (et == ETHERTYPE_VLAN) + ehl += 4; + if (m0->m_len >= ehl + (int)sizeof(*ip2)) { + ip2 = (struct ip *) + (mtod(m0, char *) + ehl); + ip2->ip_sum = 0; + ip2->ip_sum = in_cksum_hdr(ip2); + } + } + m0->m_pkthdr.csum_flags = 0; + goto tx_sg_path; + } - m0 = if_dequeue(sc->sc_ifnet); - if (m0 == NULL) { + /* Allocate SG table buffer: [prefix | SG entries] */ + sgt_buf = uma_zalloc(sc->sc_sgt_zone, M_NOWAIT); + if (sgt_buf == NULL) { + sched_unpin(); dtsec_rm_fi_free(sc, fi); - return; + m_freem(m0); + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + return (ENOBUFS); } + memset(sgt_buf, 0, sc->sc_sgt_buf_size); + + /* Store fi pointer in private data area (offset 0) */ + *(struct dtsec_rm_frame_info **)sgt_buf = fi; + /* Copy parse result into prefix */ + buf_prs = FM_PORT_GetBufferPrsResult( + sc->sc_txph, (char *)sgt_buf); + if (buf_prs != NULL) + memcpy(buf_prs, &prs, sizeof(prs)); + + /* Build SG entries at sgt_buf + tx_data_offset */ + sgt = (t_DpaaSGTE *)((char *)sgt_buf + + sc->sc_tx_data_offset); i = 0; - m = m0; psize = 0; - dsize = 0; - fi->fi_mbuf = m0; - while (m && i < DPAA_NUM_OF_SG_TABLE_ENTRY) { + for (m = m0; m != NULL; m = m->m_next) { if (m->m_len == 0) continue; - - /* - * First entry in scatter-gather table is used to keep - * pointer to frame info structure. - */ - DPAA_SGTE_SET_ADDR(&fi->fi_sgt[i], (void *)fi); - DPAA_SGTE_SET_LENGTH(&fi->fi_sgt[i], 0); - - DPAA_SGTE_SET_EXTENSION(&fi->fi_sgt[i], 0); - DPAA_SGTE_SET_FINAL(&fi->fi_sgt[i], 0); - DPAA_SGTE_SET_BPID(&fi->fi_sgt[i], 0); - DPAA_SGTE_SET_OFFSET(&fi->fi_sgt[i], 0); - i++; - dsize = m->m_len; vaddr = (vm_offset_t)m->m_data; - while (dsize > 0 && i < DPAA_NUM_OF_SG_TABLE_ENTRY) { - ssize = PAGE_SIZE - (vaddr & PAGE_MASK); - if (m->m_len < ssize) - ssize = m->m_len; - - DPAA_SGTE_SET_ADDR(&fi->fi_sgt[i], + while (dsize > 0 && + i < DPAA_NUM_OF_SG_TABLE_ENTRY) { + ssize = PAGE_SIZE - + (vaddr & PAGE_MASK); + if (ssize > dsize) + ssize = dsize; + + DPAA_SGTE_SET_ADDR(&sgt[i], (void *)vaddr); - DPAA_SGTE_SET_LENGTH(&fi->fi_sgt[i], ssize); - - DPAA_SGTE_SET_EXTENSION(&fi->fi_sgt[i], 0); - DPAA_SGTE_SET_FINAL(&fi->fi_sgt[i], 0); - DPAA_SGTE_SET_BPID(&fi->fi_sgt[i], 0); - DPAA_SGTE_SET_OFFSET(&fi->fi_sgt[i], 0); + DPAA_SGTE_SET_LENGTH(&sgt[i], ssize); + DPAA_SGTE_SET_EXTENSION(&sgt[i], 0); + DPAA_SGTE_SET_FINAL(&sgt[i], 0); + DPAA_SGTE_SET_BPID(&sgt[i], 0); + DPAA_SGTE_SET_OFFSET(&sgt[i], 0); dsize -= ssize; vaddr += ssize; psize += ssize; i++; } - if (dsize > 0) break; - - m = m->m_next; } - /* Check if SG table was constructed properly */ - if (m != NULL || dsize != 0) { + if (m != NULL || i == 0) { + /* SG table overflow or empty — drop */ + sched_unpin(); + uma_zfree(sc->sc_sgt_zone, sgt_buf); dtsec_rm_fi_free(sc, fi); m_freem(m0); - continue; + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + return (EIO); } - DPAA_SGTE_SET_FINAL(&fi->fi_sgt[i-1], 1); - - DPAA_FD_SET_ADDR(&fd, fi->fi_sgt); - DPAA_FD_SET_LENGTH(&fd, psize); - DPAA_FD_SET_FORMAT(&fd, e_DPAA_FD_FORMAT_TYPE_SHORT_MBSF); + DPAA_SGTE_SET_FINAL(&sgt[i - 1], 1); - fd.liodn = 0; - fd.bpid = 0; - fd.elion = 0; - DPAA_FD_SET_OFFSET(&fd, 0); - DPAA_FD_SET_STATUS(&fd, 0); + fi->fi_mbuf = m0; + fi->fi_tx_buf = sgt_buf; - DTSEC_UNLOCK(sc); - if (qman_fqr_enqueue(sc->sc_tx_fqr, 0, &fd) != E_OK) { + memset(&fd, 0, sizeof(fd)); + DPAA_FD_SET_ADDR(&fd, sgt_buf); + DPAA_FD_SET_LENGTH(&fd, psize); + DPAA_FD_SET_FORMAT(&fd, + e_DPAA_FD_FORMAT_TYPE_SHORT_MBSF); + DPAA_FD_SET_OFFSET(&fd, sc->sc_tx_data_offset); + DPAA_FD_SET_STATUS(&fd, + FM_FD_CMD_RPD | FM_FD_CMD_DTC); + + if (ceetm_fqid) + error = qman_enqueue_fqid(ceetm_fqid, &fd); + else + error = qman_fqr_enqueue(sc->sc_tx_fqs[cpu], + 0, &fd); + if (__predict_false(error != E_OK)) { + sched_unpin(); + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + uma_zfree(sc->sc_sgt_zone, fi->fi_tx_buf); dtsec_rm_fi_free(sc, fi); m_freem(m0); + return (EIO); } - DTSEC_LOCK(sc); + + atomic_add_int(&sc->sc_tx_inflight[cpu], 1); + sched_unpin(); + if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); + if_inc_counter(ifp, IFCOUNTER_OBYTES, psize); + return (0); } +tx_sg_path: + + /* + * Legacy SG path (no checksum offload). + * First SG entry stores fi self-reference for TX confirm. + */ + i = 0; + psize = 0; + fi->fi_mbuf = m0; + + /* SGT[0]: fi pointer (metadata, zero-length) */ + DPAA_SGTE_SET_ADDR(&fi->fi_sgt[0], (void *)fi); + DPAA_SGTE_SET_LENGTH(&fi->fi_sgt[0], 0); + DPAA_SGTE_SET_EXTENSION(&fi->fi_sgt[0], 0); + DPAA_SGTE_SET_FINAL(&fi->fi_sgt[0], 0); + DPAA_SGTE_SET_BPID(&fi->fi_sgt[0], 0); + DPAA_SGTE_SET_OFFSET(&fi->fi_sgt[0], 0); + i = 1; + + /* SGT[1..N]: data entries from mbuf chain */ + for (m = m0; m != NULL && i < DPAA_NUM_OF_SG_TABLE_ENTRY; + m = m->m_next) { + if (m->m_len == 0) + continue; + + dsize = m->m_len; + vaddr = (vm_offset_t)m->m_data; + while (dsize > 0 && i < DPAA_NUM_OF_SG_TABLE_ENTRY) { + ssize = PAGE_SIZE - (vaddr & PAGE_MASK); + if (ssize > dsize) + ssize = dsize; + + DPAA_SGTE_SET_ADDR(&fi->fi_sgt[i], + (void *)vaddr); + DPAA_SGTE_SET_LENGTH(&fi->fi_sgt[i], ssize); + DPAA_SGTE_SET_EXTENSION(&fi->fi_sgt[i], 0); + DPAA_SGTE_SET_FINAL(&fi->fi_sgt[i], 0); + DPAA_SGTE_SET_BPID(&fi->fi_sgt[i], 0); + DPAA_SGTE_SET_OFFSET(&fi->fi_sgt[i], 0); + + dsize -= ssize; + vaddr += ssize; + psize += ssize; + i++; + } + + if (dsize > 0) + break; + } + + /* Check if SG table was constructed properly */ + if (m != NULL || psize == 0) { + sched_unpin(); + dtsec_rm_fi_free(sc, fi); + m_freem(m0); + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + return (EIO); + } + + DPAA_SGTE_SET_FINAL(&fi->fi_sgt[i - 1], 1); + + memset(&fd, 0, sizeof(fd)); + DPAA_FD_SET_ADDR(&fd, fi->fi_sgt); + DPAA_FD_SET_LENGTH(&fd, psize); + DPAA_FD_SET_FORMAT(&fd, e_DPAA_FD_FORMAT_TYPE_SHORT_MBSF); + DPAA_FD_SET_OFFSET(&fd, 0); + DPAA_FD_SET_STATUS(&fd, 0); + + if (ceetm_fqid) + error = qman_enqueue_fqid(ceetm_fqid, &fd); + else + error = qman_fqr_enqueue(sc->sc_tx_fqs[cpu], 0, &fd); + if (__predict_false(error != E_OK)) { + sched_unpin(); + if_inc_counter(ifp, IFCOUNTER_OERRORS, 1); + dtsec_rm_fi_free(sc, fi); + m_freem(m0); + return (EIO); + } + + atomic_add_int(&sc->sc_tx_inflight[cpu], 1); + sched_unpin(); + if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); + if_inc_counter(ifp, IFCOUNTER_OBYTES, psize); + return (0); } /** @} */ diff --git a/sys/dev/dpaa/if_dtsec_rm.h b/sys/dev/dpaa/if_dtsec_rm.h index 28f779a11386..97a9ceb4be3d 100644 --- a/sys/dev/dpaa/if_dtsec_rm.h +++ b/sys/dev/dpaa/if_dtsec_rm.h @@ -34,7 +34,7 @@ int dtsec_rm_fm_port_rx_init(struct dtsec_softc *sc, int unit); int dtsec_rm_fm_port_tx_init(struct dtsec_softc *sc, int unit); -void dtsec_rm_if_start_locked(struct dtsec_softc *sc); +int dtsec_rm_if_transmit(if_t ifp, struct mbuf *m); int dtsec_rm_pool_rx_init(struct dtsec_softc *sc); void dtsec_rm_pool_rx_free(struct dtsec_softc *sc); @@ -46,6 +46,10 @@ int dtsec_rm_fqr_rx_init(struct dtsec_softc *sc); int dtsec_rm_fqr_tx_init(struct dtsec_softc *sc); void dtsec_rm_fqr_rx_free(struct dtsec_softc *sc); void dtsec_rm_fqr_tx_free(struct dtsec_softc *sc); +void dtsec_release_rss_fqrs(void); + +int dtsec_rm_pcd_init(struct dtsec_softc *sc); +void dtsec_rm_pcd_free(struct dtsec_softc *sc); /** @} */ #endif /* IF_DTSEC_RM_H_ */ diff --git a/sys/dev/dpaa/memac_mdio.c b/sys/dev/dpaa/memac_mdio.c new file mode 100644 index 000000000000..cda5daae4482 --- /dev/null +++ b/sys/dev/dpaa/memac_mdio.c @@ -0,0 +1,328 @@ +/*- + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * mEMAC external MDIO controller driver for FManv3 (fsl,fman-memac-mdio). + * + * On LS1046A, external PHYs are accessed through dedicated MDIO controllers + * (mdio@fc000, mdio@fd000) rather than through each MAC's internal MDIO. + * Register layout matches NXP's "xgmac_mdio" Linux driver. + * + * Modeled after fman_mdio.c (pqmdio) which handles the dTSEC MDIO layout. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "miibus_if.h" + +/* + * MDIO registers at offset 0x30 from the MDIO controller base. + * See struct memac_mii_access_mem_map in fsl_fman_memac_mii_acc.h. + */ +#define MEMAC_MDIO_CFG 0x030 +#define MEMAC_MDIO_CTRL 0x034 +#define MEMAC_MDIO_DATA 0x038 +#define MEMAC_MDIO_ADDR 0x03c + +/* MDIO_CFG bits */ +#define MDIO_CFG_BSY 0x80000000 +#define MDIO_CFG_READ_ERR 0x00000002 +#define MDIO_CFG_ENC45 0x00000040 +#define MDIO_CFG_CLK_DIV_MASK 0x0080ff80 +#define MDIO_CFG_HOLD_MASK 0x0000001c + +/* MDIO_CTRL bits */ +#define MDIO_CTL_READ 0x00008000 +#define MDIO_CTL_PHY_ADDR_SHIFT 5 + +/* MDIO_DATA bits */ +#define MDIO_DATA_BSY 0x80000000 + +#define MDIO_TIMEOUT 1000 /* iterations */ + +struct memac_mdio_softc { + device_t sc_dev; + struct mtx sc_lock; + struct resource *sc_mem; +}; + +/* + * FMan registers are big-endian. bus_read_4/write_4 on ARM64 give + * native (little-endian) access, so we byte-swap. + */ +static inline uint32_t +memac_mdio_read(struct memac_mdio_softc *sc, bus_size_t off) +{ + + return (be32toh(bus_read_4(sc->sc_mem, off))); +} + +static inline void +memac_mdio_write(struct memac_mdio_softc *sc, bus_size_t off, uint32_t val) +{ + + bus_write_4(sc->sc_mem, off, htobe32(val)); +} + +#define MDIO_LOCK() mtx_lock(&sc->sc_lock) +#define MDIO_UNLOCK() mtx_unlock(&sc->sc_lock) + +static int memac_mdio_probe(device_t dev); +static int memac_mdio_attach(device_t dev); +static int memac_mdio_detach(device_t dev); +static int memac_mdio_readreg(device_t dev, int phy, int reg); +static int memac_mdio_writereg(device_t dev, int phy, int reg, int value); + +static device_method_t memac_mdio_methods[] = { + DEVMETHOD(device_probe, memac_mdio_probe), + DEVMETHOD(device_attach, memac_mdio_attach), + DEVMETHOD(device_detach, memac_mdio_detach), + + /* MII interface */ + DEVMETHOD(miibus_readreg, memac_mdio_readreg), + DEVMETHOD(miibus_writereg, memac_mdio_writereg), + + DEVMETHOD_END +}; + +static struct ofw_compat_data memac_mdio_compat[] = { + {"fsl,fman-memac-mdio", 1}, + {NULL, 0} +}; + +static driver_t memac_mdio_driver = { + "memac_mdio", + memac_mdio_methods, + sizeof(struct memac_mdio_softc), +}; + +static int +memac_mdio_wait_free(struct memac_mdio_softc *sc) +{ + int timeout; + + for (timeout = MDIO_TIMEOUT; timeout > 0; timeout--) { + if (!(memac_mdio_read(sc, MEMAC_MDIO_CFG) & MDIO_CFG_BSY)) + return (0); + DELAY(1); + } + return (ETIMEDOUT); +} + +static int +memac_mdio_probe(device_t dev) +{ + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_search_compatible(dev, memac_mdio_compat)->ocd_str) + return (ENXIO); + + device_set_desc(dev, "Freescale mEMAC MDIO Controller"); + + return (BUS_PROBE_DEFAULT); +} + +static int +memac_mdio_attach(device_t dev) +{ + struct memac_mdio_softc *sc; + int rid; + + sc = device_get_softc(dev); + sc->sc_dev = dev; + + rid = 0; + sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->sc_mem == NULL) { + device_printf(dev, "could not allocate memory resource\n"); + return (ENXIO); + } + + OF_device_register_xref(OF_xref_from_node(ofw_bus_get_node(dev)), dev); + + mtx_init(&sc->sc_lock, device_get_nameunit(dev), + "mEMAC MDIO lock", MTX_DEF); + + return (0); +} + +static int +memac_mdio_detach(device_t dev) +{ + struct memac_mdio_softc *sc; + + sc = device_get_softc(dev); + + if (sc->sc_mem != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem); + + mtx_destroy(&sc->sc_lock); + + return (0); +} + +/* + * Clause 22 (1G) MDIO read. + * Register protocol matches fman_memac_mii_acc.c:read_phy_reg_1g(). + */ +static int +memac_mdio_readreg(device_t dev, int phy, int reg) +{ + struct memac_mdio_softc *sc; + uint32_t cfg, ctl, data; + + sc = device_get_softc(dev); + + MDIO_LOCK(); + + /* Clause 22: clear ENC45, preserve clock divider and hold */ + cfg = memac_mdio_read(sc, MEMAC_MDIO_CFG); + cfg &= (MDIO_CFG_CLK_DIV_MASK | MDIO_CFG_HOLD_MASK); + memac_mdio_write(sc, MEMAC_MDIO_CFG, cfg); + + if (memac_mdio_wait_free(sc) != 0) { + MDIO_UNLOCK(); + return (0xffff); + } + + /* Initiate read: phy in bits [9:5], register in bits [4:0] */ + ctl = ((phy & 0x1f) << MDIO_CTL_PHY_ADDR_SHIFT) | (reg & 0x1f); + ctl |= MDIO_CTL_READ; + memac_mdio_write(sc, MEMAC_MDIO_CTRL, ctl); + + if (memac_mdio_wait_free(sc) != 0) { + MDIO_UNLOCK(); + return (0xffff); + } + + /* Wait for data valid */ + { + int timeout; + for (timeout = MDIO_TIMEOUT; timeout > 0; timeout--) { + data = memac_mdio_read(sc, MEMAC_MDIO_DATA); + if (!(data & MDIO_DATA_BSY)) + break; + DELAY(1); + } + if (timeout <= 0) { + MDIO_UNLOCK(); + return (0xffff); + } + } + + /* Check for read error */ + if (memac_mdio_read(sc, MEMAC_MDIO_CFG) & MDIO_CFG_READ_ERR) { + MDIO_UNLOCK(); + return (0xffff); + } + + MDIO_UNLOCK(); + + return (data & 0xffff); +} + +/* + * Clause 22 (1G) MDIO write. + * Register protocol matches fman_memac_mii_acc.c:write_phy_reg_1g(). + */ +static int +memac_mdio_writereg(device_t dev, int phy, int reg, int value) +{ + struct memac_mdio_softc *sc; + uint32_t cfg, ctl; + + sc = device_get_softc(dev); + + MDIO_LOCK(); + + /* Clause 22: clear ENC45, preserve clock divider and hold */ + cfg = memac_mdio_read(sc, MEMAC_MDIO_CFG); + cfg &= (MDIO_CFG_CLK_DIV_MASK | MDIO_CFG_HOLD_MASK); + memac_mdio_write(sc, MEMAC_MDIO_CFG, cfg); + + if (memac_mdio_wait_free(sc) != 0) { + MDIO_UNLOCK(); + return (EIO); + } + + /* Set phy address and register */ + ctl = ((phy & 0x1f) << MDIO_CTL_PHY_ADDR_SHIFT) | (reg & 0x1f); + memac_mdio_write(sc, MEMAC_MDIO_CTRL, ctl); + + if (memac_mdio_wait_free(sc) != 0) { + MDIO_UNLOCK(); + return (EIO); + } + + /* Write data */ + memac_mdio_write(sc, MEMAC_MDIO_DATA, (uint32_t)(value & 0xffff)); + + /* Wait for write to complete */ + { + int timeout; + for (timeout = MDIO_TIMEOUT; timeout > 0; timeout--) { + if (!(memac_mdio_read(sc, MEMAC_MDIO_DATA) & + MDIO_DATA_BSY)) + break; + DELAY(1); + } + } + + MDIO_UNLOCK(); + + return (0); +} + +/* Probe before dtsec (BUS_PASS_SUPPORTDEV) so sc_mdio lookup succeeds */ +EARLY_DRIVER_MODULE(memac_mdio, fman, memac_mdio_driver, 0, 0, + BUS_PASS_SUPPORTDEV); +DRIVER_MODULE(miibus, memac_mdio, miibus_driver, 0, 0); +MODULE_DEPEND(memac_mdio, miibus, 1, 1, 1); diff --git a/sys/dev/dpaa/portals.h b/sys/dev/dpaa/portals.h index a2d6294f3d8e..3e921f6ae1b3 100644 --- a/sys/dev/dpaa/portals.h +++ b/sys/dev/dpaa/portals.h @@ -35,6 +35,8 @@ typedef struct dpaa_portal { vm_paddr_t dp_ci_pa; /* portal's CI area PA */ uint32_t dp_ce_size; /* portal's CE area size */ uint32_t dp_ci_size; /* portal's CI area size */ + vm_offset_t dp_ce_va; /* per-CPU CE virtual addr */ + vm_offset_t dp_ci_va; /* per-CPU CI virtual addr */ uintptr_t dp_intr_num; /* portal's intr. number */ } dpaa_portal_t; @@ -60,3 +62,6 @@ int qman_portals_detach(device_t); int dpaa_portal_alloc_res(device_t, struct dpaa_portals_devinfo *, int); void dpaa_portal_map_registers(struct dpaa_portals_softc *); +#ifdef __aarch64__ +void dpaa_portal_map_registers_cpu(struct dpaa_portals_softc *, int); +#endif diff --git a/sys/dev/dpaa/portals_common.c b/sys/dev/dpaa/portals_common.c index 5ef28570b481..58e63d0f4054 100644 --- a/sys/dev/dpaa/portals_common.c +++ b/sys/dev/dpaa/portals_common.c @@ -39,7 +39,12 @@ #include #include +#ifdef __powerpc__ #include +#endif +#ifdef __aarch64__ +#include +#endif #include #include @@ -70,6 +75,11 @@ dpaa_portal_alloc_res(device_t dev, struct dpaa_portals_devinfo *di, int cpu) /* Cache enabled area */ rle = resource_list_find(res, SYS_RES_MEMORY, 0); sc->sc_rrid[0] = 0; + device_printf(dev, "CE alloc: pa=%#lx-%#lx size=%#lx dp_pa=%#lx\n", + (unsigned long)(rle->start + sc->sc_dp_pa), + (unsigned long)(rle->end + sc->sc_dp_pa), + (unsigned long)rle->count, + (unsigned long)sc->sc_dp_pa); sc->sc_rres[0] = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid[0], rle->start + sc->sc_dp_pa, rle->end + sc->sc_dp_pa, rle->count, RF_ACTIVE); @@ -78,11 +88,37 @@ dpaa_portal_alloc_res(device_t dev, struct dpaa_portals_devinfo *di, int cpu) "Could not allocate cache enabled memory.\n"); return (ENXIO); } + device_printf(dev, "CE mapped: va=%#lx\n", + (unsigned long)rman_get_bushandle(sc->sc_rres[0])); +#ifdef __powerpc__ tlb1_set_entry(rman_get_bushandle(sc->sc_rres[0]), rle->start + sc->sc_dp_pa, rle->count, _TLB_ENTRY_MEM); +#endif +#ifdef __aarch64__ + /* + * ARM64: The CE portal region must be Normal Non-Cacheable. + * Linux maps this as MEMREMAP_WC (pgprot_writecombine = + * Normal-NC) on ARM64. Stores go directly to the portal + * hardware without cache involvement. dc zva works on + * Normal-NC (zeros written as burst), dc cvac is a no-op. + */ + pmap_change_attr( + (vm_offset_t)rman_get_bushandle(sc->sc_rres[0]), + rle->count, VM_MEMATTR_UNCACHEABLE); + device_printf(dev, "CE pa_verify: va=%#lx -> pa=%#lx " + "(expected %#lx)\n", + (unsigned long)rman_get_bushandle(sc->sc_rres[0]), + (unsigned long)pmap_kextract( + (vm_offset_t)rman_get_bushandle(sc->sc_rres[0])), + (unsigned long)(rle->start + sc->sc_dp_pa)); +#endif /* Cache inhibited area */ rle = resource_list_find(res, SYS_RES_MEMORY, 1); sc->sc_rrid[1] = 1; + device_printf(dev, "CI alloc: pa=%#lx-%#lx size=%#lx\n", + (unsigned long)(rle->start + sc->sc_dp_pa), + (unsigned long)(rle->end + sc->sc_dp_pa), + (unsigned long)rle->count); sc->sc_rres[1] = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->sc_rrid[1], rle->start + sc->sc_dp_pa, rle->end + sc->sc_dp_pa, rle->count, RF_ACTIVE); @@ -93,8 +129,16 @@ dpaa_portal_alloc_res(device_t dev, struct dpaa_portals_devinfo *di, int cpu) sc->sc_rrid[0], sc->sc_rres[0]); return (ENXIO); } + device_printf(dev, "CI mapped: va=%#lx\n", + (unsigned long)rman_get_bushandle(sc->sc_rres[1])); +#ifdef __powerpc__ tlb1_set_entry(rman_get_bushandle(sc->sc_rres[1]), rle->start + sc->sc_dp_pa, rle->count, _TLB_ENTRY_IO); +#endif + sc->sc_dp[cpu].dp_ce_va = + rman_get_bushandle(sc->sc_rres[0]); + sc->sc_dp[cpu].dp_ci_va = + rman_get_bushandle(sc->sc_rres[1]); sc->sc_dp[cpu].dp_regs_mapped = 1; } /* Acquire portal's CE_PA and CI_PA */ @@ -153,15 +197,64 @@ dpaa_portal_map_registers(struct dpaa_portals_softc *sc) if (sc->sc_dp[cpu].dp_regs_mapped) goto out; +#ifdef __powerpc__ + /* PowerPC: remap the shared VA to this CPU's portal PA via TLB */ tlb1_set_entry(rman_get_bushandle(sc->sc_rres[0]), sc->sc_dp[cpu].dp_ce_pa, sc->sc_dp[cpu].dp_ce_size, _TLB_ENTRY_MEM); tlb1_set_entry(rman_get_bushandle(sc->sc_rres[1]), sc->sc_dp[cpu].dp_ci_pa, sc->sc_dp[cpu].dp_ci_size, _TLB_ENTRY_IO); + sc->sc_dp[cpu].dp_ce_va = rman_get_bushandle(sc->sc_rres[0]); + sc->sc_dp[cpu].dp_ci_va = rman_get_bushandle(sc->sc_rres[1]); +#endif +#ifdef __aarch64__ + /* + * ARM64: Each CPU needs its own VA→PA mapping because we can't + * remap a shared VA per-CPU like PowerPC does with TLB1 entries. + * Map each portal's CE as Normal Non-Cacheable, CI as Device. + */ + if (sc->sc_dp[cpu].dp_ce_pa == 0) { + printf("dpaa_portal_map_registers: cpu %u has no portal allocated\n", cpu); + goto out; + } + sc->sc_dp[cpu].dp_ce_va = (vm_offset_t)pmap_mapdev_attr( + sc->sc_dp[cpu].dp_ce_pa, sc->sc_dp[cpu].dp_ce_size, + VM_MEMATTR_UNCACHEABLE); + sc->sc_dp[cpu].dp_ci_va = (vm_offset_t)pmap_mapdev( + sc->sc_dp[cpu].dp_ci_pa, sc->sc_dp[cpu].dp_ci_size); +#endif sc->sc_dp[cpu].dp_regs_mapped = 1; out: sched_unpin(); } + +#ifdef __aarch64__ +/* + * Map portal CE/CI registers for a specific CPU. + * Unlike dpaa_portal_map_registers() which uses PCPU_GET(cpuid), + * this takes an explicit CPU parameter so it can be called from + * any CPU context during early boot (before smp_started=1). + * pmap_mapdev_attr/pmap_mapdev create global kernel VA mappings. + */ +void +dpaa_portal_map_registers_cpu(struct dpaa_portals_softc *sc, int cpu) +{ + + if (sc->sc_dp[cpu].dp_regs_mapped) + return; + if (sc->sc_dp[cpu].dp_ce_pa == 0) { + printf("dpaa_portal_map_registers_cpu: " + "cpu %d has no portal allocated\n", cpu); + return; + } + sc->sc_dp[cpu].dp_ce_va = (vm_offset_t)pmap_mapdev_attr( + sc->sc_dp[cpu].dp_ce_pa, sc->sc_dp[cpu].dp_ce_size, + VM_MEMATTR_UNCACHEABLE); + sc->sc_dp[cpu].dp_ci_va = (vm_offset_t)pmap_mapdev( + sc->sc_dp[cpu].dp_ci_pa, sc->sc_dp[cpu].dp_ci_size); + sc->sc_dp[cpu].dp_regs_mapped = 1; +} +#endif diff --git a/sys/dev/dpaa/qman.c b/sys/dev/dpaa/qman.c index e9159f25d74b..c51511ee455a 100644 --- a/sys/dev/dpaa/qman.c +++ b/sys/dev/dpaa/qman.c @@ -25,6 +25,8 @@ */ #include +#include "opt_dpaa.h" + #include #include #include @@ -37,16 +39,33 @@ #include #include #include +#include #include #include +#ifdef __powerpc__ #include +#endif + +#ifdef __aarch64__ +#include +#include +#include +#include +#endif +#include "bman.h" #include "qman.h" #include "portals.h" +#ifdef QMAN_CEETM_SUPPORT +#include "qman_ceetm.h" +#endif + +#define FQD_ENTRY_SIZE 64 /* matches NCSW qm.h */ + extern struct dpaa_portals_softc *qp_sc; -static struct qman_softc *qman_sc; +struct qman_softc *qman_sc; extern t_Handle qman_portal_setup(struct qman_softc *qsc); @@ -126,11 +145,28 @@ e_RxStoreResponse qman_received_frame_callback(t_Handle app, t_Handle qm_fqr, t_Handle qm_portal, uint32_t fqid_offset, t_DpaaFD *frame) { - struct qman_softc *sc; - - sc = app; - - device_printf(sc->sc_dev, "dummy callback for received frame.\n"); + static volatile int dflt_cb_count = 0; + t_Handle pool; + int n = ++dflt_cb_count; + + /* + * This callback fires for frames dequeued from FQs that have no + * registered callback (e.g., FQIDs created by FMC PCD schemes). + * When fqr==NULL, fqid_offset is the raw FQID, not an offset. + * + * Return the buffer to its BMan pool to prevent pool depletion. + */ + pool = bman_pool_for_bpid(frame->bpid); + if (pool != NULL) { + bman_put_buffer(pool, DPAA_FD_GET_ADDR(frame)); + } else if (n <= 50 || (n % 500) == 0) { + printf("qman: DEFAULT CB #%d: fqr=%p fqid=%u bpid=%u " + "status=0x%08x len=%u addr=%p [NO POOL]\n", + n, qm_fqr, fqid_offset, frame->bpid, + DPAA_FD_GET_STATUS(frame), + DPAA_FD_GET_LENGTH(frame), + DPAA_FD_GET_ADDR(frame)); + } return (e_RX_STORE_RESPONSE_CONTINUE); } @@ -144,14 +180,114 @@ qman_rejected_frame_callback(t_Handle app, t_Handle qm_fqr, t_Handle qm_portal, uint32_t fqid_offset, t_DpaaFD *frame, t_QmRejectedFrameInfo *qm_rejected_frame_info) { + static int ern_count; struct qman_softc *sc; sc = app; - device_printf(sc->sc_dev, "dummy callback for rejected frame.\n"); + if (ern_count++ < 20) + device_printf(sc->sc_dev, + "ERN: rejected frame fqid_off=%u rc=%d\n", + fqid_offset, + qm_rejected_frame_info ? + qm_rejected_frame_info->rejectionCode : -1); return (e_RX_STORE_RESPONSE_CONTINUE); } +/* + * Map a DT reserved-memory region for QBMan use. + * + * If the reserved-memory node has a 'reg' property (fixed physical address), + * the region is already excluded from phys_avail[] by early boot code + * (fdt_foreach_reserved_mem + no-map). We map it with pmap_mapdev_attr() + * as write-back cacheable — LS1046A CCI-400 provides HW coherency. + * + * Falls back to contigmalloc for nodes with only 'size' (no fixed address). + * + * Returns mapped VA, or NULL on failure. *out_size receives the size. + */ +#ifdef __aarch64__ +MALLOC_DECLARE(M_NETCOMMSW); + +void *qbman_alloc_reserved_mem(device_t dev, phandle_t mem_node, + uint32_t *out_size, const char *name); +void * +qbman_alloc_reserved_mem(device_t dev, phandle_t mem_node, + uint32_t *out_size, const char *name) +{ + pcell_t reg[4]; /* addr_hi, addr_lo, size_hi, size_lo */ + pcell_t cells[2]; + uint64_t pa, size, alignment; + void *va; + + /* + * Prefer fixed physical address from 'reg' property. + * DT reserved-memory with no-map + reg is excluded from phys_avail[] + * at early boot, so this memory is never touched by the VM system. + */ + if (OF_getencprop(mem_node, "reg", reg, sizeof(reg)) == + sizeof(reg)) { + pa = ((uint64_t)reg[0] << 32) | reg[1]; + size = ((uint64_t)reg[2] << 32) | reg[3]; + + if (size == 0 || size > 64 * 1024 * 1024) { + device_printf(dev, "%s: invalid reg size %#jx\n", + name, (uintmax_t)size); + return (NULL); + } + + va = pmap_mapdev_attr(pa, size, VM_MEMATTR_WRITE_BACK); + if (va == NULL) { + device_printf(dev, + "%s: pmap_mapdev_attr(%#jx, %#jx) failed\n", + name, (uintmax_t)pa, (uintmax_t)size); + return (NULL); + } + + memset(va, 0, size); + + *out_size = (uint32_t)size; + device_printf(dev, + "%s: %uMB at pa=%#jx va=%p (no-map reserved)\n", + name, (unsigned)(size >> 20), (uintmax_t)pa, va); + return (va); + } + + /* + * Fallback: dynamic allocation from 'size' property. + */ + if (OF_getencprop(mem_node, "size", cells, sizeof(cells)) <= 0) { + device_printf(dev, "%s: no 'reg' or 'size' property\n", name); + return (NULL); + } + size = ((uint64_t)cells[0] << 32) | cells[1]; + + if (OF_getencprop(mem_node, "alignment", cells, sizeof(cells)) <= 0) + alignment = size; + else + alignment = ((uint64_t)cells[0] << 32) | cells[1]; + + if (size == 0 || size > 64 * 1024 * 1024) { + device_printf(dev, "%s: invalid size %#jx\n", + name, (uintmax_t)size); + return (NULL); + } + + va = contigmalloc(size, M_NETCOMMSW, M_WAITOK | M_ZERO, + 0, 0xFFFFFFFFULL, alignment, 0); + if (va == NULL) { + device_printf(dev, "%s: contigmalloc(%#jx, align=%#jx) failed\n", + name, (uintmax_t)size, (uintmax_t)alignment); + return (NULL); + } + + *out_size = (uint32_t)size; + device_printf(dev, "%s: %uMB at %p (contigmalloc, align=%#jx)\n", + name, (unsigned)(size >> 20), va, (uintmax_t)alignment); + return (va); +} +#endif + int qman_attach(device_t dev) { @@ -191,9 +327,12 @@ qman_attach(device_t dev) if (qp_sc == NULL) goto err; +#ifndef __aarch64__ dpaa_portal_map_registers(qp_sc); +#endif /* Initialize QMan */ + memset(&qp, 0, sizeof(qp)); qp.guestId = NCSW_MASTER_ID; qp.baseAddress = rman_get_bushandle(sc->sc_rres); qp.swPortalsBaseAddress = rman_get_bushandle(qp_sc->sc_rres[0]); @@ -209,6 +348,48 @@ qman_attach(device_t dev) qp.partCgsBase = 0; qp.partNumOfCgs = 0; +#ifdef __aarch64__ + /* + * Parse DT memory-region phandles to allocate FQD and PFDR memory + * from reserved-memory nodes, matching Linux's qbman_init_private_mem(). + * DT: memory-region = <&qman_fqd &qman_pfdr>; + * qman-fqd: size=0x800000 (8MB) → 131072 FQIDs + * qman-pfdr: size=0x2000000 (32MB) → 524288 PFDRs + */ + { + phandle_t node, mem_node; + pcell_t mem_handles[2]; + + node = ofw_bus_get_node(dev); + if (node > 0 && OF_getencprop(node, "memory-region", + (void *)mem_handles, sizeof(mem_handles)) == + sizeof(mem_handles)) { + /* Index 0 = FQD, Index 1 = PFDR */ + mem_node = OF_node_from_xref(mem_handles[0]); + if (mem_node > 0) { + qp.p_FqdBase = qbman_alloc_reserved_mem(dev, + mem_node, &qp.fqdSize, "FQD"); + if (qp.p_FqdBase != NULL) { + qp.totalNumOfFqids = + qp.fqdSize / FQD_ENTRY_SIZE; + qp.partNumOfFqids = + qp.totalNumOfFqids; + } + } + /* Index 1 = PFDR */ + mem_node = OF_node_from_xref(mem_handles[1]); + if (mem_node > 0) { + qp.p_PfdrBase = + qbman_alloc_reserved_mem(dev, + mem_node, &qp.pfdrSize, "PFDR"); + } + } else { + device_printf(dev, + "no memory-region property, using defaults\n"); + } + } +#endif + sc->sc_qh = QM_Config(&qp); if (sc->sc_qh == NULL) { device_printf(dev, "could not be configured\n"); @@ -230,9 +411,34 @@ qman_attach(device_t dev) device_printf(dev, "Hardware version: %d.%d.\n", rev.majorRev, rev.minorRev); +#ifndef __aarch64__ + qman_portal_setup(sc); +#endif + sched_unpin(); - qman_portal_setup(sc); +#ifdef __aarch64__ + /* + * Initialize portals for all CPUs from boot CPU. + * ARM64: pmap_mapdev creates global kernel VA, no sched_bind needed. + * SDEST configured per-portal via QM_PORTAL_ConfigStash. + * All portals use NAPI-style interrupt handling (FILTER + taskqueue). + */ + for (int cpu = 0; cpu < mp_ncpus; cpu++) { + if (qman_portal_init_cpu(sc, cpu) == NULL) + device_printf(dev, + "could not setup QMan portal on CPU %d\n", cpu); + else + device_printf(dev, + "QMan portal %d initialized (SDEST=%d)\n", + cpu, cpu / 2); + } + +#ifdef QMAN_CEETM_SUPPORT + /* Initialize CEETM if DT node is present */ + qman_ceetm_init(); +#endif +#endif return (0); @@ -360,7 +566,73 @@ qman_fqr_create(uint32_t fqids_num, e_QmFQChannel channel, uint8_t wq, goto err; } - sc->sc_fqr_cpu[QM_FQR_GetFqid(fqrh)] = PCPU_GET(cpuid); + if (QM_FQR_GetFqid(fqrh) < QMAN_MAX_FQIDS) + sc->sc_fqr_cpu[QM_FQR_GetFqid(fqrh)] = PCPU_GET(cpuid); + + sched_unpin(); + + return (fqrh); + +err: + sched_unpin(); + + return (NULL); +} + +t_Handle +qman_fqr_create_ctx(uint32_t fqids_num, e_QmFQChannel channel, uint8_t wq, + bool force_fqid, uint32_t fqid_or_align, bool prefer_in_cache, + t_QmContextA *p_context_a, t_QmContextB *p_context_b) +{ + struct qman_softc *sc; + t_QmFqrParams fqr; + t_Handle fqrh, portal; + + sc = qman_sc; + + sched_pin(); + + portal = qman_portal_setup(sc); + if (portal == NULL) { + device_printf(sc->sc_dev, "could not setup QMan portal\n"); + goto err; + } + + fqr.h_Qm = sc->sc_qh; + fqr.h_QmPortal = portal; + fqr.initParked = FALSE; + fqr.holdActive = FALSE; + fqr.preferInCache = prefer_in_cache; + + fqr.useContextAForStash = FALSE; + fqr.p_ContextA = p_context_a; + fqr.p_ContextB = p_context_b; + + fqr.channel = channel; + fqr.wq = wq; + fqr.shadowMode = FALSE; + fqr.numOfFqids = fqids_num; + + fqr.useForce = force_fqid; + if (force_fqid) + fqr.qs.frcQ.fqid = fqid_or_align; + else + fqr.qs.nonFrcQs.align = fqid_or_align; + + fqr.congestionAvoidanceEnable = FALSE; + fqr.congestionAvoidanceParams.h_QmCg = 0; + fqr.congestionAvoidanceParams.overheadAccountingLength = 0; + fqr.congestionAvoidanceParams.fqTailDropThreshold = 0; + + fqrh = QM_FQR_Create(&fqr); + if (fqrh == NULL) { + device_printf(sc->sc_dev, + "could not create Frame Queue Range (ctx)\n"); + goto err; + } + + if (QM_FQR_GetFqid(fqrh) < QMAN_MAX_FQIDS) + sc->sc_fqr_cpu[QM_FQR_GetFqid(fqrh)] = PCPU_GET(cpuid); sched_unpin(); @@ -379,9 +651,11 @@ qman_fqr_free(t_Handle fqr) t_Error error; sc = qman_sc; - thread_lock(curthread); - sched_bind(curthread, sc->sc_fqr_cpu[QM_FQR_GetFqid(fqr)]); - thread_unlock(curthread); + if (QM_FQR_GetFqid(fqr) < QMAN_MAX_FQIDS) { + thread_lock(curthread); + sched_bind(curthread, sc->sc_fqr_cpu[QM_FQR_GetFqid(fqr)]); + thread_unlock(curthread); + } error = QM_FQR_Free(fqr); @@ -443,6 +717,30 @@ qman_fqr_enqueue(t_Handle fqr, uint32_t fqid_off, t_DpaaFD *frame) return (error); } +t_Error +qman_enqueue_fqid(uint32_t fqid, t_DpaaFD *frame) +{ + struct qman_softc *sc; + t_Error error; + t_Handle portal; + + sc = qman_sc; + sched_pin(); + + portal = qman_portal_setup(sc); + if (portal == NULL) { + device_printf(sc->sc_dev, "could not setup QMan portal\n"); + sched_unpin(); + return (E_NOT_SUPPORTED); + } + + error = QM_PORTAL_EnqueueFqid(portal, fqid, frame); + + sched_unpin(); + + return (error); +} + uint32_t qman_fqr_get_counter(t_Handle fqr, uint32_t fqid_off, e_QmFqrCounters counter) @@ -545,7 +843,64 @@ qman_poll(e_QmPortalPollSource source) } /* - * TODO: add polling and/or congestion support. + * QMan diagnostic counters — expose QM_GetCounter to dtsec driver. */ +uint32_t +qman_get_pfdr_in_use(void) +{ + struct qman_softc *sc; + + sc = qman_sc; + if (sc == NULL || sc->sc_qh == NULL) + return (0); + + return (QM_GetCounter(sc->sc_qh, e_QM_COUNTERS_PFDR_IN_USE)); +} + +uint32_t +qman_get_sfdr_in_use(void) +{ + struct qman_softc *sc; + + sc = qman_sc; + if (sc == NULL || sc->sc_qh == NULL) + return (0); + + return (QM_GetCounter(sc->sc_qh, e_QM_COUNTERS_SFDR_IN_USE)); +} + +#ifdef __aarch64__ +void +qman_ccsr_diag(void) +{ + struct qman_softc *sc; + uint32_t err_isr, ecsr, ecir, pfdr_fpc, sfdr_in_use; + uint32_t idle_stat, dcp0_cfg, dcp0_dlm_avg; + uint32_t dcp_dd_ihrsr, dcp_dd_hasr; + + sc = qman_sc; + if (sc == NULL || sc->sc_qh == NULL) + return; + + QM_ErrorDiag(sc->sc_qh, &err_isr, &ecsr, &ecir, &pfdr_fpc, + &sfdr_in_use, &idle_stat, &dcp0_cfg, &dcp0_dlm_avg, + &dcp_dd_ihrsr, &dcp_dd_hasr); + + printf("qman: CCSR err_isr=0x%08x ecsr=0x%08x ecir=0x%08x\n", + err_isr, ecsr, ecir); + if (ecir) { + printf("qman: ecir: %s portal %u fqid %u\n", + (ecir & 0x20000000) ? "DCP" : "SWP", + (ecir >> 24) & 0x1f, + ecir & 0x00ffffff); + } + printf("qman: pfdr_free=%u sfdr_in_use=%u idle=0x%08x\n", + pfdr_fpc, sfdr_in_use, idle_stat); + printf("qman: dcp0: cfg=0x%08x dlm_avg=0x%08x\n", + dcp0_cfg, dcp0_dlm_avg); + printf("qman: dcp_dd: ihrsr=0x%08x hasr=0x%08x\n", + dcp_dd_ihrsr, dcp_dd_hasr); +} +#endif /** @} */ diff --git a/sys/dev/dpaa/qman.h b/sys/dev/dpaa/qman.h index 815ef1f6d33a..f6b54a97a8f0 100644 --- a/sys/dev/dpaa/qman.h +++ b/sys/dev/dpaa/qman.h @@ -39,7 +39,7 @@ /** * Maximum number of frame queues in all QMans. */ -#define QMAN_MAX_FQIDS 16 +#define QMAN_MAX_FQIDS 4096 /** * Pool channel common to all software portals. @@ -85,6 +85,8 @@ struct qman_softc { * @group QMan bus interface * @{ */ +extern struct qman_softc *qman_sc; + int qman_attach(device_t dev); int qman_detach(device_t dev); int qman_suspend(device_t dev); @@ -185,6 +187,16 @@ t_Error qman_fqr_register_cb(t_Handle fqr, t_QmReceivedFrameCallback *callback, */ t_Error qman_fqr_enqueue(t_Handle fqr, uint32_t fqid_off, t_DpaaFD *frame); +/** + * Enqueue a frame to a specific FQID (bypassing FQR). + * Used for CEETM frame queues managed by CEETM hardware. + * + * @param fqid Target Frame Queue ID. + * @param frame A frame to be enqueued. + * @return E_OK on success; error code otherwise. + */ +t_Error qman_enqueue_fqid(uint32_t fqid, t_DpaaFD *frame); + /** * Get one of the FQR counter's value. * @@ -239,6 +251,40 @@ e_RxStoreResponse qman_rejected_frame_callback(t_Handle app, t_Handle qm_fqr, t_Handle qm_portal, uint32_t fqid_offset, t_DpaaFD *frame, t_QmRejectedFrameInfo *qm_rejected_frame_info); +/** + * Create Frame Queue Range with explicit contextA/B. + * + * Same as qman_fqr_create() but allows setting raw FQD contextA and + * contextB fields. Used by CDX to create forwarding TX FQs with + * OVFQ/EBD bits set for hardware flow offload. + * + * @param fqids_num Number of frame queues in the range. + * @param channel Dedicated channel (e.g. FMan TX port). + * @param wq Work Queue Number within the channel. + * @param force_fqid If TRUE, force allocation of specific FQID. + * @param fqid_or_align FQID if force_fqid, alignment otherwise. + * @param prefer_in_cache If TRUE, prefer QMan internal cache. + * @param p_context_a Raw contextA (8 bytes), or NULL. + * @param p_context_b Raw contextB (4 bytes), or NULL. + * @return NCSW FQR handle, or NULL on failure. + */ +t_Handle qman_fqr_create_ctx(uint32_t fqids_num, e_QmFQChannel channel, + uint8_t wq, bool force_fqid, uint32_t fqid_or_align, + bool prefer_in_cache, t_QmContextA *p_context_a, + t_QmContextB *p_context_b); + +uint32_t qman_get_pfdr_in_use(void); +uint32_t qman_get_sfdr_in_use(void); + +#ifdef __aarch64__ +t_Handle qman_portal_init_cpu(struct qman_softc *, int); +void qman_portal_dqrr_diag(void); +void qman_ccsr_diag(void); +void qman_portal_quiesce(void); +struct mbuf; +void qman_rx_defer(struct mbuf *); +#endif + /** @} */ #endif /* QMAN_H */ diff --git a/sys/dev/dpaa/qman_ceetm.c b/sys/dev/dpaa/qman_ceetm.c new file mode 100644 index 000000000000..5e5cf4f64e4f --- /dev/null +++ b/sys/dev/dpaa/qman_ceetm.c @@ -0,0 +1,1874 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Mono Technologies Inc. + * + * QMan CEETM (Channel Egress Enhancement and Traffic Management) driver + * for FreeBSD. Ported from the Linux NXP QBMan SDK driver. + * + * Original Linux source: + * drivers/staging/fsl_qbman/qman_high.c (API implementation) + * drivers/staging/fsl_qbman/qman_driver.c (DT init) + * drivers/staging/fsl_qbman/qman_config.c (CCSR registers) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* NCSW internal headers — accessible because we compile with DPAA_COMPILE_CMD */ +#include "qm.h" +#include "qman_low.h" + +/* FreeBSD DPAA headers */ +#include "qman.h" +#include "qman_ceetm.h" + +MALLOC_DECLARE(M_NETCOMMSW); + +/* ================================================================ + * Global State + * ================================================================ */ + +uint8_t qman_num_ceetms; +struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX]; + +/* QMan clock frequency (Hz) — set during init from DT or default */ +static uint32_t qman_clk; + +/* CEETM update period in nanoseconds (default 1µs) */ +#define CEETM_UPDATE_PERIOD_NS 1000 + +/* LFQID and Channel allocators — simple bitmaps */ +#define CEETM_MAX_LFQIDS 4096 +#define CEETM_MAX_CHANNELS 32 + +struct ceetm_pool { + uint32_t base; + uint32_t count; + uint32_t bitmap[CEETM_MAX_LFQIDS / 32]; /* max size */ +}; + +static struct ceetm_pool ceetm_lfqid_pool[QMAN_CEETM_MAX]; +static struct ceetm_pool ceetm_channel_pool[QMAN_CEETM_MAX]; +static struct mtx ceetm_alloc_lock; + +static void +ceetm_pool_seed(struct ceetm_pool *pool, uint32_t base, uint32_t count) +{ + pool->base = base; + pool->count = count; + memset(pool->bitmap, 0, sizeof(pool->bitmap)); +} + +static int +ceetm_pool_alloc(struct ceetm_pool *pool, uint32_t *id) +{ + unsigned int i; + + mtx_lock(&ceetm_alloc_lock); + for (i = 0; i < pool->count; i++) { + if (!(pool->bitmap[i / 32] & (1U << (i % 32)))) { + pool->bitmap[i / 32] |= (1U << (i % 32)); + *id = pool->base + i; + mtx_unlock(&ceetm_alloc_lock); + return (0); + } + } + mtx_unlock(&ceetm_alloc_lock); + return (ENOSPC); +} + +static void +ceetm_pool_free(struct ceetm_pool *pool, uint32_t id) +{ + unsigned int idx; + + if (id < pool->base || id >= pool->base + pool->count) + return; + idx = id - pool->base; + mtx_lock(&ceetm_alloc_lock); + pool->bitmap[idx / 32] &= ~(1U << (idx % 32)); + mtx_unlock(&ceetm_alloc_lock); +} + +/* ================================================================ + * Portal MC (Management Command) Helpers + * + * Pattern: sched_pin → get portal → NCSW_PLOCK → mc_start → + * fill command → mc_commit → spin on mc_result → + * check result → PUNLOCK → sched_unpin + * ================================================================ */ + +/* + * Get the current CPU's NCSW QM portal, with preemption pinned. + * Caller must have called sched_pin() first. + */ +static inline t_QmPortal * +ceetm_get_portal(void) +{ + return ((t_QmPortal *)qman_sc->sc_qph[PCPU_GET(cpuid)]); +} + +/* + * Get the t_Qm handle for CCSR register access. + */ +static inline t_Qm * +ceetm_get_qm(void) +{ + return ((t_Qm *)qman_sc->sc_qh); +} + +/* + * CCSR register read/write — for registers not in t_QmRegs struct. + * Uses byte offset from QMan CCSR base. + */ +static inline uint32_t +ceetm_ccsr_read(uint32_t offset) +{ + volatile uint32_t *reg; + + reg = (volatile uint32_t *)((uint8_t *)ceetm_get_qm()->p_QmRegs + + offset); + return (GET_UINT32(*reg)); +} + +static inline void +ceetm_ccsr_write(uint32_t offset, uint32_t val) +{ + volatile uint32_t *reg; + + reg = (volatile uint32_t *)((uint8_t *)ceetm_get_qm()->p_QmRegs + + offset); + WRITE_UINT32(*reg, val); +} + +/* Rounding helper for integer division */ +static inline uint64_t +ceetm_div_round(uint64_t n, uint64_t d, int rounding) +{ + if (rounding < 0) + return (n / d); /* round down */ + else if (rounding > 0) + return ((n + d - 1) / d); /* round up */ + else + return ((n + d / 2) / d); /* round nearest */ +} + +/* ================================================================ + * CCSR Register Access Functions + * ================================================================ */ + +int +qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, uint16_t sub_portal) +{ + uint32_t dcp_cfg; + t_Qm *qm; + + if (qman_num_ceetms == 0) + return (ENODEV); + + qm = ceetm_get_qm(); + dcp_cfg = GET_UINT32(qm->p_QmRegs->dcpConfRegs[portal].cfg); + dcp_cfg |= QM_SP_ENABLE_CEETM(sub_portal); + WRITE_UINT32(qm->p_QmRegs->dcpConfRegs[portal].cfg, dcp_cfg); + return (0); +} + +int +qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, uint16_t sub_portal) +{ + uint32_t dcp_cfg; + t_Qm *qm; + + if (qman_num_ceetms == 0) + return (ENODEV); + + qm = ceetm_get_qm(); + dcp_cfg = GET_UINT32(qm->p_QmRegs->dcpConfRegs[portal].cfg); + dcp_cfg &= ~QM_SP_ENABLE_CEETM(sub_portal); + WRITE_UINT32(qm->p_QmRegs->dcpConfRegs[portal].cfg, dcp_cfg); + return (0); +} + +int +qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num) +{ + if (qman_num_ceetms == 0) + return (ENODEV); + *num = ceetm_ccsr_read(QM_REG_CEETM_XSFDR_IN_USE); + return (0); +} + +static int +qman_ceetm_set_prescaler(enum qm_dc_portal portal) +{ + uint64_t temp; + uint16_t pres; + + if (qman_clk == 0) + return (EINVAL); + + /* + * PRES = (2^22 * 10^9 / update_period_ns) / qman_clk + * Split to avoid 64-bit overflow: + * temp = 0x400000 * 100 + * temp /= (update_period_ns / 10000000) + * But simpler: temp = (0x400000ULL * 1000000000ULL) / (period * clk) + */ + temp = 0x400000ULL * 100; + temp /= CEETM_UPDATE_PERIOD_NS; + temp *= 10000000; + temp /= qman_clk; + pres = (uint16_t)temp; + + ceetm_ccsr_write(QM_REG_CEETM_CFG_IDX, portal); + ceetm_ccsr_write(QM_REG_CEETM_CFG_PRES, pres); + return (0); +} + +static int +qman_ceetm_get_prescaler(uint16_t *pres) +{ + if (qman_num_ceetms == 0) + return (ENODEV); + *pres = (uint16_t)ceetm_ccsr_read(QM_REG_CEETM_CFG_PRES); + return (0); +} + +/* ================================================================ + * Low-Level MC Command Wrappers + * + * Each sends a single MC command and returns the result code. + * The command struct is filled by the caller; these functions handle + * portal lock/unlock, commit, and result polling. + * ================================================================ */ + +/* + * Generic MC command execution pattern. + * cmd points to the filled command struct (excluding verb byte). + * cmd_size is the size of the command struct. + * verb is the CEETM verb code. + * result is filled with the 64-byte result on success. + * Returns 0 on success, error code otherwise. + */ +static int +ceetm_mc_command(uint8_t verb, const void *cmd, size_t cmd_size, + void *result, size_t result_size) +{ + t_QmPortal *portal; + struct qm_portal *low; + struct qm_mc_command *mcc; + struct qm_mc_result *mcr; + struct qm_mcr_ceetm_generic *gen; + int timeout; + + sched_pin(); + portal = ceetm_get_portal(); + if (portal == NULL) { + sched_unpin(); + return (ENXIO); + } + + low = portal->p_LowQmPortal; + NCSW_PLOCK(portal); + + mcc = qm_mc_start(low); + if (mcc == NULL) { + PUNLOCK(portal); + sched_unpin(); + return (EBUSY); + } + + /* Copy command data after the verb byte */ + memcpy((uint8_t *)mcc + 1, cmd, cmd_size); + qm_mc_commit(low, verb); + + /* Poll for result */ + timeout = 10000; + do { + mcr = qm_mc_result(low); + if (mcr != NULL) + break; + cpu_spinwait(); + } while (--timeout > 0); + + PUNLOCK(portal); + sched_unpin(); + + if (timeout == 0) { + printf("ceetm: MC command 0x%02x timeout\n", verb); + return (ETIMEDOUT); + } + + /* Check result code */ + gen = (struct qm_mcr_ceetm_generic *)mcr; + if (gen->result != QM_MCR_RESULT_OK) { + printf("ceetm: MC command 0x%02x failed, result=0x%02x\n", + verb, gen->result); + return (EIO); + } + + if (result != NULL && result_size > 0) + memcpy(result, mcr, result_size); + return (0); +} + +/* Convenience wrappers for config (no result needed) and query (result needed) */ +static int +ceetm_mc_config(uint8_t verb, const void *cmd, size_t cmd_size) +{ + return (ceetm_mc_command(verb, cmd, cmd_size, NULL, 0)); +} + +static int +ceetm_mc_query(uint8_t verb, const void *cmd, size_t cmd_size, + void *result, size_t result_size) +{ + return (ceetm_mc_command(verb, cmd, cmd_size, result, result_size)); +} + +/* ================================================================ + * Pure Math Functions + * ================================================================ */ + +int +qman_ceetm_bps2tokenrate(uint64_t bps, struct qm_ceetm_rate *token_rate, + int rounding) +{ + uint16_t pres; + uint64_t temp, qman_freq; + int ret; + + ret = qman_ceetm_get_prescaler(&pres); + if (ret) + return (EINVAL); + if (pres == 0) + return (EINVAL); + + qman_freq = (uint64_t)qman_clk; + if (qman_freq == 0) + return (EINVAL); + + /* + * N = (bps * 2^32) / (PRES * QHz) + * Split into two 16-bit shifts to avoid overflow: + * N = (((bps << 16) / PRES) << 16) / QHz + */ + temp = ceetm_div_round(bps << 16, pres, rounding); + temp = ceetm_div_round(temp << 16, qman_freq, rounding); + token_rate->whole = (uint32_t)(temp >> 13); + token_rate->fraction = (uint32_t)(temp & 0x1FFF); + return (0); +} + +int +qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, + uint64_t *bps, int rounding) +{ + uint16_t pres; + uint64_t temp, qman_freq; + int ret; + + ret = qman_ceetm_get_prescaler(&pres); + if (ret) + return (EINVAL); + + qman_freq = (uint64_t)qman_clk; + if (qman_freq == 0) + return (EINVAL); + + /* + * bps = N * PRES * QHz / 2^32 + * Split: temp = (PRES * QHz) >> 16, then bps = (temp * N) >> 16 + */ + temp = ceetm_div_round(qman_freq * pres, (uint64_t)1 << 16, rounding); + temp *= ((uint64_t)token_rate->whole << 13) + token_rate->fraction; + *bps = ceetm_div_round(temp, (uint64_t)1 << 16, rounding); + return (0); +} + +/* + * Find greatest common divisor. + */ +static uint32_t +ceetm_gcd(uint32_t a, uint32_t b) +{ + while (b != 0) { + uint32_t t = b; + b = a % b; + a = t; + } + return (a); +} + +int +qman_ceetm_ratio2wbfs(uint32_t numerator, uint32_t denominator, + struct qm_ceetm_weight_code *weight_code, int rounding) +{ + uint32_t g; + uint64_t n, d; + int best_err; + + if (denominator == 0 || numerator == 0) + return (EINVAL); + + /* Reduce fraction */ + g = ceetm_gcd(numerator, denominator); + n = numerator / g; + d = denominator / g; + + /* + * WBFS weight = 2^(x+6) / (64-y) + * Find (x,y) such that n/d ~ 2^(x+6)/(64-y) + * i.e., y ~ 64 - d * 2^(x+6) / n + */ + best_err = INT_MAX; + weight_code->x = 0; + weight_code->y = 0; + + for (uint8_t x = 0; x <= 7; x++) { + int64_t ideal_64_minus_y; + int64_t y_candidate; + int err; + + ideal_64_minus_y = (int64_t)((d * ((uint64_t)1 << (x + 6))) / + n); + y_candidate = 64 - ideal_64_minus_y; + + if (y_candidate < 0) + y_candidate = 0; + if (y_candidate > 31) + y_candidate = 31; + + /* Check error */ + err = (int)((int64_t)n * (64 - y_candidate) - + (int64_t)d * ((uint64_t)1 << (x + 6))); + if (err < 0) + err = -err; + + if (err < best_err) { + best_err = err; + weight_code->x = x; + weight_code->y = (uint8_t)y_candidate; + if (err == 0) + break; + } + } + return (0); +} + +int +qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code, + uint32_t *numerator, uint32_t *denominator) +{ + uint32_t g; + + /* weight = 2^(x+6) / (64-y) */ + *numerator = 1U << (weight_code->x + 6); + *denominator = 64 - weight_code->y; + if (*denominator == 0) + return (EINVAL); + + g = ceetm_gcd(*numerator, *denominator); + *numerator /= g; + *denominator /= g; + return (0); +} + +/* ================================================================ + * Sub-Portal Management + * ================================================================ */ + +int +qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, enum qm_dc_portal dcp_idx, + unsigned int sp_idx) +{ + struct qm_ceetm_sp *p; + + if (dcp_idx >= qman_num_ceetms) + return (EINVAL); + if (sp_idx < qman_ceetms[dcp_idx].sp_range[0] || + sp_idx >= qman_ceetms[dcp_idx].sp_range[0] + + qman_ceetms[dcp_idx].sp_range[1]) + return (EINVAL); + + TAILQ_FOREACH(p, &qman_ceetms[dcp_idx].sub_portals, entries) { + if (p->idx == sp_idx) { + if (p->is_claimed) + return (EBUSY); + p->is_claimed = 1; + *sp = p; + return (0); + } + } + return (ENOENT); +} + +int +qman_ceetm_sp_release(struct qm_ceetm_sp *sp) +{ + if (sp == NULL) + return (EINVAL); + if (sp->lni != NULL) + qman_sp_disable_ceetm_mode(sp->dcp_idx, sp->idx); + sp->lni = NULL; + sp->is_claimed = 0; + return (0); +} + +int +qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, struct qm_ceetm_lni *lni) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_config cmd; + int ret; + + if (sp == NULL || lni == NULL) + return (EINVAL); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16(CEETM_COMMAND_SP_MAPPING | sp->idx); + cmd.dcpid = sp->dcp_idx; + cmd.sp_mapping.map_flags = lni->idx & 0x07; + + ret = ceetm_mc_config(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG, + &cmd, sizeof(cmd)); + if (ret) + return (ret); + + sp->lni = lni; + lni->sp = sp; + qman_sp_enable_ceetm_mode(sp->dcp_idx, sp->idx); + return (0); +} + +int +qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, unsigned int *lni_idx) +{ + if (sp == NULL || sp->lni == NULL) + return (EINVAL); + *lni_idx = sp->lni->idx; + return (0); +} + +/* ================================================================ + * LNI Management + * ================================================================ */ + +int +qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, enum qm_dc_portal dcp_idx, + unsigned int lni_idx) +{ + struct qm_ceetm_lni *p; + + if (dcp_idx >= qman_num_ceetms) + return (EINVAL); + + TAILQ_FOREACH(p, &qman_ceetms[dcp_idx].lnis, entries) { + if (p->idx == lni_idx) { + if (p->is_claimed) + return (EBUSY); + p->is_claimed = 1; + *lni = p; + return (0); + } + } + return (ENOENT); +} + +int +qman_ceetm_lni_release(struct qm_ceetm_lni *lni) +{ + if (lni == NULL) + return (EINVAL); + if (lni->shaper_enable) + qman_ceetm_lni_disable_shaper(lni); + lni->sp = NULL; + lni->is_claimed = 0; + return (0); +} + +int +qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled, int oal) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_config cmd; + int ret; + + if (lni == NULL) + return (EINVAL); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16(CEETM_COMMAND_LNI_SHAPER | lni->idx); + cmd.dcpid = lni->dcp_idx; + cmd.shaper_config.cpl_oal = (coupled ? 0x80 : 0) | (oal & 0x1F); + cmd.shaper_config.crtcr_hi = (lni->cr_token_rate.whole >> 8) & 0x07; + cmd.shaper_config.crtcr_lo = htobe16( + ((lni->cr_token_rate.whole & 0xFF) << 13) | + (lni->cr_token_rate.fraction & 0x1FFF)); + cmd.shaper_config.ertcr_hi = (lni->er_token_rate.whole >> 8) & 0x07; + cmd.shaper_config.ertcr_lo = htobe16( + ((lni->er_token_rate.whole & 0xFF) << 13) | + (lni->er_token_rate.fraction & 0x1FFF)); + cmd.shaper_config.crtbl = htobe16(lni->cr_token_bucket_limit); + cmd.shaper_config.ertbl = htobe16(lni->er_token_bucket_limit); + + ret = ceetm_mc_config(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG, + &cmd, sizeof(cmd)); + if (ret) + return (ret); + + lni->shaper_enable = 1; + lni->shaper_couple = coupled; + lni->oal = oal; + return (0); +} + +int +qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_config cmd; + int ret; + + if (lni == NULL) + return (EINVAL); + + /* Write zero rates to disable shaper */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16(CEETM_COMMAND_LNI_SHAPER | lni->idx); + cmd.dcpid = lni->dcp_idx; + + ret = ceetm_mc_config(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG, + &cmd, sizeof(cmd)); + if (ret) + return (ret); + + lni->shaper_enable = 0; + return (0); +} + +int +qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni) +{ + if (lni == NULL) + return (0); + return (lni->shaper_enable); +} + +int +qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni, + const struct qm_ceetm_rate *cr, uint16_t cr_limit) +{ + if (lni == NULL || cr == NULL) + return (EINVAL); + + lni->cr_token_rate = *cr; + lni->cr_token_bucket_limit = cr_limit; + + /* If shaper is already enabled, re-program */ + if (lni->shaper_enable) + return (qman_ceetm_lni_enable_shaper(lni, + lni->shaper_couple, lni->oal)); + return (0); +} + +int +qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni, + const struct qm_ceetm_rate *er, uint16_t er_limit) +{ + if (lni == NULL || er == NULL) + return (EINVAL); + + lni->er_token_rate = *er; + lni->er_token_bucket_limit = er_limit; + + if (lni->shaper_enable) + return (qman_ceetm_lni_enable_shaper(lni, + lni->shaper_couple, lni->oal)); + return (0); +} + +int +qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni, + struct qm_ceetm_rate *cr, uint16_t *cr_limit) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_query qcmd; + struct qm_mcr_ceetm_mapping_shaper_tcfc_query qr; + uint32_t tcr; + int ret; + + if (lni == NULL) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cid = htobe16(CEETM_COMMAND_LNI_SHAPER | lni->idx); + qcmd.dcpid = lni->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + tcr = ((uint32_t)qr.shaper_query.crtcr_hi << 16) | + be16toh(qr.shaper_query.crtcr_lo); + if (cr != NULL) { + cr->whole = tcr >> 13; + cr->fraction = tcr & 0x1FFF; + } + if (cr_limit != NULL) + *cr_limit = be16toh(qr.shaper_query.crtbl); + return (0); +} + +int +qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni, + struct qm_ceetm_rate *er, uint16_t *er_limit) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_query qcmd; + struct qm_mcr_ceetm_mapping_shaper_tcfc_query qr; + uint32_t tcr; + int ret; + + if (lni == NULL) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cid = htobe16(CEETM_COMMAND_LNI_SHAPER | lni->idx); + qcmd.dcpid = lni->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + tcr = ((uint32_t)qr.shaper_query.ertcr_hi << 16) | + be16toh(qr.shaper_query.ertcr_lo); + if (er != NULL) { + er->whole = tcr >> 13; + er->fraction = tcr & 0x1FFF; + } + if (er_limit != NULL) + *er_limit = be16toh(qr.shaper_query.ertbl); + return (0); +} + +/* ================================================================ + * Channel Management + * ================================================================ */ + +int +qman_ceetm_channel_claim(struct qm_ceetm_channel **channel, + struct qm_ceetm_lni *lni) +{ + struct qm_ceetm_channel *ch; + struct qm_mcc_ceetm_mapping_shaper_tcfc_config cmd; + uint32_t ch_id; + int ret; + + if (lni == NULL || channel == NULL) + return (EINVAL); + + ret = ceetm_pool_alloc(&ceetm_channel_pool[lni->dcp_idx], &ch_id); + if (ret) + return (ret); + + ch = malloc(sizeof(*ch), M_NETCOMMSW, M_NOWAIT | M_ZERO); + if (ch == NULL) { + ceetm_pool_free(&ceetm_channel_pool[lni->dcp_idx], ch_id); + return (ENOMEM); + } + + ch->idx = ch_id; + ch->lni = lni; + ch->lni_idx = lni->idx; + ch->dcp_idx = lni->dcp_idx; + TAILQ_INIT(&ch->class_queues); + TAILQ_INIT(&ch->ccgs); + + /* Map channel to LNI */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16(CEETM_COMMAND_CHANNEL_MAPPING | ch->idx); + cmd.dcpid = ch->dcp_idx; + cmd.channel_mapping.map_flags = lni->idx & 0x07; + + ret = ceetm_mc_config(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG, + &cmd, sizeof(cmd)); + if (ret) { + ceetm_pool_free(&ceetm_channel_pool[lni->dcp_idx], ch_id); + free(ch, M_NETCOMMSW); + return (ret); + } + + TAILQ_INSERT_TAIL(&lni->channels, ch, entries); + *channel = ch; + return (0); +} + +int +qman_ceetm_channel_release(struct qm_ceetm_channel *channel) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_config cmd; + + if (channel == NULL) + return (EINVAL); + + /* Disable shaper if enabled */ + if (channel->shaper_enable) + qman_ceetm_channel_disable_shaper(channel); + + /* Clear channel mapping */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16(CEETM_COMMAND_CHANNEL_MAPPING | channel->idx); + cmd.dcpid = channel->dcp_idx; + ceetm_mc_config(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG, + &cmd, sizeof(cmd)); + + ceetm_pool_free(&ceetm_channel_pool[channel->dcp_idx], channel->idx); + if (channel->lni != NULL) + TAILQ_REMOVE(&channel->lni->channels, channel, entries); + free(channel, M_NETCOMMSW); + return (0); +} + +int +qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel, + int coupled) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_config cmd; + int ret; + + if (channel == NULL) + return (EINVAL); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16(CEETM_COMMAND_CHANNEL_SHAPER | channel->idx); + cmd.dcpid = channel->dcp_idx; + cmd.shaper_config.cpl_oal = coupled ? 0x80 : 0; + cmd.shaper_config.crtcr_hi = + (channel->cr_token_rate.whole >> 8) & 0x07; + cmd.shaper_config.crtcr_lo = htobe16( + ((channel->cr_token_rate.whole & 0xFF) << 13) | + (channel->cr_token_rate.fraction & 0x1FFF)); + cmd.shaper_config.ertcr_hi = + (channel->er_token_rate.whole >> 8) & 0x07; + cmd.shaper_config.ertcr_lo = htobe16( + ((channel->er_token_rate.whole & 0xFF) << 13) | + (channel->er_token_rate.fraction & 0x1FFF)); + cmd.shaper_config.crtbl = htobe16(channel->cr_token_bucket_limit); + cmd.shaper_config.ertbl = htobe16(channel->er_token_bucket_limit); + + ret = ceetm_mc_config(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG, + &cmd, sizeof(cmd)); + if (ret) + return (ret); + + channel->shaper_enable = 1; + channel->shaper_couple = coupled; + return (0); +} + +int +qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_config cmd; + int ret; + + if (channel == NULL) + return (EINVAL); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16(CEETM_COMMAND_CHANNEL_SHAPER | channel->idx); + cmd.dcpid = channel->dcp_idx; + + ret = ceetm_mc_config(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG, + &cmd, sizeof(cmd)); + if (ret) + return (ret); + + channel->shaper_enable = 0; + return (0); +} + +int +qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel) +{ + if (channel == NULL) + return (0); + return (channel->shaper_enable); +} + +int +qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel, + const struct qm_ceetm_rate *cr, uint16_t cr_limit) +{ + if (channel == NULL || cr == NULL) + return (EINVAL); + + channel->cr_token_rate = *cr; + channel->cr_token_bucket_limit = cr_limit; + + if (channel->shaper_enable) + return (qman_ceetm_channel_enable_shaper(channel, + channel->shaper_couple)); + return (0); +} + +int +qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel, + const struct qm_ceetm_rate *er, uint16_t er_limit) +{ + if (channel == NULL || er == NULL) + return (EINVAL); + + channel->er_token_rate = *er; + channel->er_token_bucket_limit = er_limit; + + if (channel->shaper_enable) + return (qman_ceetm_channel_enable_shaper(channel, + channel->shaper_couple)); + return (0); +} + +int +qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel, + struct qm_ceetm_rate *cr, uint16_t *cr_limit) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_query qcmd; + struct qm_mcr_ceetm_mapping_shaper_tcfc_query qr; + uint32_t tcr; + int ret; + + if (channel == NULL) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cid = htobe16(CEETM_COMMAND_CHANNEL_SHAPER | channel->idx); + qcmd.dcpid = channel->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + tcr = ((uint32_t)qr.shaper_query.crtcr_hi << 16) | + be16toh(qr.shaper_query.crtcr_lo); + if (cr != NULL) { + cr->whole = tcr >> 13; + cr->fraction = tcr & 0x1FFF; + } + if (cr_limit != NULL) + *cr_limit = be16toh(qr.shaper_query.crtbl); + return (0); +} + +int +qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel, + struct qm_ceetm_rate *er, uint16_t *er_limit) +{ + struct qm_mcc_ceetm_mapping_shaper_tcfc_query qcmd; + struct qm_mcr_ceetm_mapping_shaper_tcfc_query qr; + uint32_t tcr; + int ret; + + if (channel == NULL) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cid = htobe16(CEETM_COMMAND_CHANNEL_SHAPER | channel->idx); + qcmd.dcpid = channel->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + tcr = ((uint32_t)qr.shaper_query.ertcr_hi << 16) | + be16toh(qr.shaper_query.ertcr_lo); + if (er != NULL) { + er->whole = tcr >> 13; + er->fraction = tcr & 0x1FFF; + } + if (er_limit != NULL) + *er_limit = be16toh(qr.shaper_query.ertbl); + return (0); +} + +/* ================================================================ + * Channel Class Scheduler (Group/Eligibility) + * ================================================================ */ + +int +qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, + int group_b, unsigned int prio_a, unsigned int prio_b) +{ + struct qm_mcc_ceetm_class_scheduler_config cmd; + int ret; + + if (channel == NULL) + return (EINVAL); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cqcid = htobe16(channel->idx); + cmd.dcpid = channel->dcp_idx; + cmd.gpc = (prio_a & CEETM_GPC_PRIO_A_MASK); + if (group_b) { + cmd.gpc |= CEETM_GPC_COMBINE_FLAG; + cmd.gpc |= (prio_b << CEETM_GPC_PRIO_B_SHIFT) & + CEETM_GPC_PRIO_B_MASK; + } + + ret = ceetm_mc_config(QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG, + &cmd, sizeof(cmd)); + return (ret); +} + +int +qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, + int *group_b, unsigned int *prio_a, unsigned int *prio_b) +{ + struct qm_mcc_ceetm_class_scheduler_query qcmd; + struct qm_mcr_ceetm_class_scheduler_query qr; + int ret; + + if (channel == NULL) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cqcid = htobe16(channel->idx); + qcmd.dcpid = channel->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_CLASS_SCHEDULER_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + if (group_b != NULL) + *group_b = (qr.gpc & CEETM_GPC_COMBINE_FLAG) ? 1 : 0; + if (prio_a != NULL) + *prio_a = qr.gpc & CEETM_GPC_PRIO_A_MASK; + if (prio_b != NULL) + *prio_b = (qr.gpc & CEETM_GPC_PRIO_B_MASK) >> + CEETM_GPC_PRIO_B_SHIFT; + return (0); +} + +int +qman_ceetm_channel_set_group_cr_eligibility( + struct qm_ceetm_channel *channel, int group_b, int cre) +{ + struct qm_mcc_ceetm_class_scheduler_query qcmd; + struct qm_mcr_ceetm_class_scheduler_query qr; + struct qm_mcc_ceetm_class_scheduler_config cmd; + uint16_t mask; + int ret; + + if (channel == NULL) + return (EINVAL); + + /* Query current state */ + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cqcid = htobe16(channel->idx); + qcmd.dcpid = channel->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_CLASS_SCHEDULER_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + mask = be16toh(qr.crem); + if (group_b) { + if (cre) + mask |= (1 << 9); + else + mask &= ~(1 << 9); + } else { + if (cre) + mask |= (1 << 8); + else + mask &= ~(1 << 8); + } + + memset(&cmd, 0, sizeof(cmd)); + cmd.cqcid = htobe16(channel->idx); + cmd.dcpid = channel->dcp_idx; + cmd.gpc = qr.gpc; + cmd.crem = htobe16(mask); + cmd.erem = qr.erem; + memcpy(cmd.w, qr.w, 8); + + return (ceetm_mc_config(QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG, + &cmd, sizeof(cmd))); +} + +int +qman_ceetm_channel_set_group_er_eligibility( + struct qm_ceetm_channel *channel, int group_b, int ere) +{ + struct qm_mcc_ceetm_class_scheduler_query qcmd; + struct qm_mcr_ceetm_class_scheduler_query qr; + struct qm_mcc_ceetm_class_scheduler_config cmd; + uint16_t mask; + int ret; + + if (channel == NULL) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cqcid = htobe16(channel->idx); + qcmd.dcpid = channel->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_CLASS_SCHEDULER_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + mask = be16toh(qr.erem); + if (group_b) { + if (ere) + mask |= (1 << 9); + else + mask &= ~(1 << 9); + } else { + if (ere) + mask |= (1 << 8); + else + mask &= ~(1 << 8); + } + + memset(&cmd, 0, sizeof(cmd)); + cmd.cqcid = htobe16(channel->idx); + cmd.dcpid = channel->dcp_idx; + cmd.gpc = qr.gpc; + cmd.crem = qr.crem; + cmd.erem = htobe16(mask); + memcpy(cmd.w, qr.w, 8); + + return (ceetm_mc_config(QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG, + &cmd, sizeof(cmd))); +} + +int +qman_ceetm_channel_set_cq_cr_eligibility( + struct qm_ceetm_channel *channel, unsigned int idx, int cre) +{ + struct qm_mcc_ceetm_class_scheduler_query qcmd; + struct qm_mcr_ceetm_class_scheduler_query qr; + struct qm_mcc_ceetm_class_scheduler_config cmd; + uint16_t mask; + int ret; + + if (channel == NULL || idx > 7) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cqcid = htobe16(channel->idx); + qcmd.dcpid = channel->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_CLASS_SCHEDULER_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + mask = be16toh(qr.crem); + if (cre) + mask |= (1 << idx); + else + mask &= ~(1 << idx); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cqcid = htobe16(channel->idx); + cmd.dcpid = channel->dcp_idx; + cmd.gpc = qr.gpc; + cmd.crem = htobe16(mask); + cmd.erem = qr.erem; + memcpy(cmd.w, qr.w, 8); + + return (ceetm_mc_config(QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG, + &cmd, sizeof(cmd))); +} + +int +qman_ceetm_channel_set_cq_er_eligibility( + struct qm_ceetm_channel *channel, unsigned int idx, int ere) +{ + struct qm_mcc_ceetm_class_scheduler_query qcmd; + struct qm_mcr_ceetm_class_scheduler_query qr; + struct qm_mcc_ceetm_class_scheduler_config cmd; + uint16_t mask; + int ret; + + if (channel == NULL || idx > 7) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cqcid = htobe16(channel->idx); + qcmd.dcpid = channel->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_CLASS_SCHEDULER_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + mask = be16toh(qr.erem); + if (ere) + mask |= (1 << idx); + else + mask &= ~(1 << idx); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cqcid = htobe16(channel->idx); + cmd.dcpid = channel->dcp_idx; + cmd.gpc = qr.gpc; + cmd.crem = qr.crem; + cmd.erem = htobe16(mask); + memcpy(cmd.w, qr.w, 8); + + return (ceetm_mc_config(QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG, + &cmd, sizeof(cmd))); +} + +/* ================================================================ + * Class Queue Management + * ================================================================ */ + +int +qman_ceetm_cq_claim(struct qm_ceetm_cq **cq, + struct qm_ceetm_channel *channel, unsigned int idx, + struct qm_ceetm_ccg *ccg) +{ + struct qm_ceetm_cq *p; + struct qm_mcc_ceetm_cq_config cmd; + int ret; + + if (channel == NULL || cq == NULL || idx > 7) + return (EINVAL); + + /* Check not already claimed */ + TAILQ_FOREACH(p, &channel->class_queues, entries) { + if (p->idx == idx) + return (EBUSY); + } + + p = malloc(sizeof(*p), M_NETCOMMSW, M_NOWAIT | M_ZERO); + if (p == NULL) + return (ENOMEM); + + p->idx = idx; + p->parent = channel; + p->ccg = ccg; + TAILQ_INIT(&p->bound_lfqids); + + /* Configure CQ */ + memset(&cmd, 0, sizeof(cmd)); + cmd.cqid = htobe16((channel->idx << 4) | idx); + cmd.dcpid = channel->dcp_idx; + if (ccg != NULL) + cmd.ccgid = htobe16((ccg->parent->idx << 4) | ccg->idx); + + ret = ceetm_mc_config(QM_CEETM_VERB_CQ_CONFIG, &cmd, sizeof(cmd)); + if (ret) { + free(p, M_NETCOMMSW); + return (ret); + } + + TAILQ_INSERT_TAIL(&channel->class_queues, p, entries); + p->is_claimed = 1; + *cq = p; + return (0); +} + +int +qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq, + struct qm_ceetm_channel *channel, unsigned int idx, + struct qm_ceetm_ccg *ccg) +{ + /* Group A uses CQ indices 8-15 */ + if (idx < 8 || idx > 15) + return (EINVAL); + return (qman_ceetm_cq_claim(cq, channel, idx, ccg)); +} + +int +qman_ceetm_cq_release(struct qm_ceetm_cq *cq) +{ + if (cq == NULL) + return (EINVAL); + + TAILQ_REMOVE(&cq->parent->class_queues, cq, entries); + free(cq, M_NETCOMMSW); + return (0); +} + +int +qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq, + struct qm_ceetm_weight_code *weight_code) +{ + struct qm_mcc_ceetm_class_scheduler_query qcmd; + struct qm_mcr_ceetm_class_scheduler_query qr; + struct qm_mcc_ceetm_class_scheduler_config cmd; + int ret; + unsigned int w_idx; + + if (cq == NULL || weight_code == NULL) + return (EINVAL); + if (cq->idx > 15) + return (EINVAL); + + /* Query current scheduler state */ + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cqcid = htobe16(cq->parent->idx); + qcmd.dcpid = cq->parent->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_CLASS_SCHEDULER_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + /* Weight index: CQ 0-7 → w[0-7], CQ 8-15 → w[0-7] (group B offset) */ + w_idx = cq->idx & 0x07; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cqcid = htobe16(cq->parent->idx); + cmd.dcpid = cq->parent->dcp_idx; + cmd.gpc = qr.gpc; + cmd.crem = qr.crem; + cmd.erem = qr.erem; + memcpy(cmd.w, qr.w, 8); + + /* Encode weight: w = (y << 3) | x */ + cmd.w[w_idx] = (weight_code->y << 3) | weight_code->x; + + return (ceetm_mc_config(QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG, + &cmd, sizeof(cmd))); +} + +int +qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq, + struct qm_ceetm_weight_code *weight_code) +{ + struct qm_mcc_ceetm_class_scheduler_query qcmd; + struct qm_mcr_ceetm_class_scheduler_query qr; + int ret; + unsigned int w_idx; + + if (cq == NULL || weight_code == NULL) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.cqcid = htobe16(cq->parent->idx); + qcmd.dcpid = cq->parent->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_CLASS_SCHEDULER_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + w_idx = cq->idx & 0x07; + weight_code->y = (qr.w[w_idx] >> 3) & 0x1F; + weight_code->x = qr.w[w_idx] & 0x07; + return (0); +} + +/* ================================================================ + * LFQ (Logical Frame Queue) Management + * ================================================================ */ + +#define CEETM_LFQMT_LFQID_MSB 0xF00000 +#define CEETM_LFQMT_LFQID_LSB 0x000FFF + +int +qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq, struct qm_ceetm_cq *cq) +{ + struct qm_ceetm_lfq *p; + struct qm_mcc_ceetm_lfqmt_config cmd; + uint32_t lfqid, lfqid_packed; + int ret; + + if (cq == NULL || lfq == NULL) + return (EINVAL); + + ret = ceetm_pool_alloc(&ceetm_lfqid_pool[cq->parent->dcp_idx], + &lfqid); + if (ret) + return (ret); + + p = malloc(sizeof(*p), M_NETCOMMSW, M_NOWAIT | M_ZERO); + if (p == NULL) { + ceetm_pool_free(&ceetm_lfqid_pool[cq->parent->dcp_idx], + lfqid); + return (ENOMEM); + } + + p->idx = lfqid; + p->dctidx = (uint16_t)(lfqid & CEETM_LFQMT_LFQID_LSB); + p->parent = cq; + + /* Configure LFQMT */ + memset(&cmd, 0, sizeof(cmd)); + lfqid_packed = CEETM_LFQMT_LFQID_MSB | + ((uint32_t)cq->parent->dcp_idx << 16) | + (lfqid & CEETM_LFQMT_LFQID_LSB); + cmd.lfqid_hi = (lfqid_packed >> 16) & 0xFF; + cmd.lfqid_lo = htobe16(lfqid_packed & 0xFFFF); + cmd.cqid = htobe16((cq->parent->idx << 4) | cq->idx); + cmd.dctidx = htobe16(p->dctidx); + + ret = ceetm_mc_config(QM_CEETM_VERB_LFQMT_CONFIG, + &cmd, sizeof(cmd)); + if (ret) { + ceetm_pool_free(&ceetm_lfqid_pool[cq->parent->dcp_idx], + lfqid); + free(p, M_NETCOMMSW); + return (ret); + } + + TAILQ_INSERT_TAIL(&cq->bound_lfqids, p, entries); + *lfq = p; + return (0); +} + +int +qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq) +{ + enum qm_dc_portal dcp_idx; + + if (lfq == NULL) + return (EINVAL); + + dcp_idx = lfq->parent->parent->dcp_idx; + ceetm_pool_free(&ceetm_lfqid_pool[dcp_idx], lfq->idx); + TAILQ_REMOVE(&lfq->parent->bound_lfqids, lfq, entries); + free(lfq, M_NETCOMMSW); + return (0); +} + +int +qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, + uint64_t context_a, uint32_t context_b) +{ + struct qm_mcc_ceetm_dct_config cmd; + + if (lfq == NULL) + return (EINVAL); + + lfq->context_a = context_a; + lfq->context_b = context_b; + + memset(&cmd, 0, sizeof(cmd)); + cmd.dctidx = htobe16(lfq->dctidx); + cmd.dcpid = lfq->parent->parent->dcp_idx; + cmd.context_b = htobe32(context_b); + cmd.context_a = htobe64(context_a); + + return (ceetm_mc_config(QM_CEETM_VERB_DCT_CONFIG, + &cmd, sizeof(cmd))); +} + +int +qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, + uint64_t *context_a, uint32_t *context_b) +{ + struct qm_mcc_ceetm_dct_query qcmd; + struct qm_mcr_ceetm_dct_query qr; + int ret; + + if (lfq == NULL) + return (EINVAL); + + memset(&qcmd, 0, sizeof(qcmd)); + qcmd.dctidx = htobe16(lfq->dctidx); + qcmd.dcpid = lfq->parent->parent->dcp_idx; + + ret = ceetm_mc_query(QM_CEETM_VERB_DCT_QUERY, + &qcmd, sizeof(qcmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + if (context_a != NULL) + *context_a = be64toh(qr.context_a); + if (context_b != NULL) + *context_b = be32toh(qr.context_b); + return (0); +} + +int +qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq) +{ + if (lfq == NULL || fq == NULL) + return (EINVAL); + + memset(fq, 0, sizeof(*fq)); + fq->fqid = lfq->idx; + fq->flags = 0; /* QMAN_FQ_FLAG_NO_MODIFY equivalent */ + return (0); +} + +/* ================================================================ + * CCG (Class Congestion Group) Management + * ================================================================ */ + +#define MAX_CCG_IDX 0x000F + +int +qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg, + struct qm_ceetm_channel *channel, unsigned int idx, + void (*cb)(void *, int), void *cb_ctx) +{ + struct qm_ceetm_ccg *p; + + if (channel == NULL || ccg == NULL || idx > MAX_CCG_IDX) + return (EINVAL); + + /* Check not already claimed */ + TAILQ_FOREACH(p, &channel->ccgs, entries) { + if (p->idx == idx) + return (EBUSY); + } + + p = malloc(sizeof(*p), M_NETCOMMSW, M_NOWAIT | M_ZERO); + if (p == NULL) + return (ENOMEM); + + p->idx = idx; + p->parent = channel; + TAILQ_INSERT_TAIL(&channel->ccgs, p, entries); + *ccg = p; + return (0); +} + +int +qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg) +{ + if (ccg == NULL) + return (EINVAL); + + TAILQ_REMOVE(&ccg->parent->ccgs, ccg, entries); + free(ccg, M_NETCOMMSW); + return (0); +} + +int +qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, uint16_t we_mask, + struct qm_mcc_ceetm_ccgr_config *opts) +{ + struct qm_mcc_ceetm_ccgr_config cmd; + + if (ccg == NULL || opts == NULL) + return (EINVAL); + + memcpy(&cmd, opts, sizeof(cmd)); + cmd.ccgrid = htobe16(CEETM_CCGR_CM_CONFIGURE | + (ccg->parent->idx << 4) | ccg->idx); + cmd.dcpid = ccg->parent->dcp_idx; + cmd.we_mask = htobe16(we_mask); + + return (ceetm_mc_config(QM_CEETM_VERB_CCGR_CONFIG, + &cmd, sizeof(cmd))); +} + +int +qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, + uint32_t flags, uint64_t *frame_count, uint64_t *byte_count) +{ + struct qm_mcc_ceetm_statistics_query_write cmd; + struct qm_mcr_ceetm_statistics_query qr; + uint16_t command_type; + int ret; + + if (ccg == NULL) + return (EINVAL); + + if (flags & 0x01) + command_type = CEETM_QUERY_REJECT_CLEAR_STATISTICS; + else + command_type = CEETM_QUERY_REJECT_STATISTICS; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16((ccg->parent->idx << 4) | ccg->idx); + cmd.dcpid = ccg->parent->dcp_idx; + cmd.ct = command_type; + + ret = ceetm_mc_query(QM_CEETM_VERB_STATISTICS_QUERY_WRITE, + &cmd, sizeof(cmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + /* Decode 40-bit frame count and 48-bit byte count */ + if (frame_count != NULL) { + *frame_count = ((uint64_t)qr.frm_cnt[0] << 32) | + ((uint64_t)qr.frm_cnt[1] << 24) | + ((uint64_t)qr.frm_cnt[2] << 16) | + ((uint64_t)qr.frm_cnt[3] << 8) | + (uint64_t)qr.frm_cnt[4]; + } + if (byte_count != NULL) { + *byte_count = ((uint64_t)qr.byte_cnt[0] << 40) | + ((uint64_t)qr.byte_cnt[1] << 32) | + ((uint64_t)qr.byte_cnt[2] << 24) | + ((uint64_t)qr.byte_cnt[3] << 16) | + ((uint64_t)qr.byte_cnt[4] << 8) | + (uint64_t)qr.byte_cnt[5]; + } + return (0); +} + +/* ================================================================ + * Statistics/Query + * ================================================================ */ + +int +qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid, + struct qm_mcr_ceetm_cq_query *cq_query) +{ + struct qm_mcc_ceetm_cq_query cmd; + + if (cq_query == NULL) + return (EINVAL); + + memset(&cmd, 0, sizeof(cmd)); + cmd.cqid = htobe16(cqid); + cmd.dcpid = dcpid; + + return (ceetm_mc_query(QM_CEETM_VERB_CQ_QUERY, + &cmd, sizeof(cmd), cq_query, sizeof(*cq_query))); +} + +int +qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, + uint32_t flags, uint64_t *frame_count, uint64_t *byte_count) +{ + struct qm_mcc_ceetm_statistics_query_write cmd; + struct qm_mcr_ceetm_statistics_query qr; + uint16_t command_type; + int ret; + + if (cq == NULL) + return (EINVAL); + + if (flags & 0x01) + command_type = CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS; + else + command_type = CEETM_QUERY_DEQUEUE_STATISTICS; + + memset(&cmd, 0, sizeof(cmd)); + cmd.cid = htobe16((cq->parent->idx << 4) | cq->idx); + cmd.dcpid = cq->parent->dcp_idx; + cmd.ct = command_type; + + ret = ceetm_mc_query(QM_CEETM_VERB_STATISTICS_QUERY_WRITE, + &cmd, sizeof(cmd), &qr, sizeof(qr)); + if (ret) + return (ret); + + if (frame_count != NULL) { + *frame_count = ((uint64_t)qr.frm_cnt[0] << 32) | + ((uint64_t)qr.frm_cnt[1] << 24) | + ((uint64_t)qr.frm_cnt[2] << 16) | + ((uint64_t)qr.frm_cnt[3] << 8) | + (uint64_t)qr.frm_cnt[4]; + } + if (byte_count != NULL) { + *byte_count = ((uint64_t)qr.byte_cnt[0] << 40) | + ((uint64_t)qr.byte_cnt[1] << 32) | + ((uint64_t)qr.byte_cnt[2] << 24) | + ((uint64_t)qr.byte_cnt[3] << 16) | + ((uint64_t)qr.byte_cnt[4] << 8) | + (uint64_t)qr.byte_cnt[5]; + } + return (0); +} + +/* ================================================================ + * MAPPING_SHAPER_TCFC direct command + * ================================================================ */ + +int +qman_ceetm_configure_mapping_shaper_tcfc( + struct qm_mcc_ceetm_mapping_shaper_tcfc_config *opts) +{ + if (opts == NULL) + return (EINVAL); + return (ceetm_mc_config(QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG, + opts, sizeof(*opts))); +} + +/* ================================================================ + * Initialization — called from qman_attach() if DT node present + * ================================================================ */ + +int +qman_ceetm_init(void) +{ + phandle_t qman_node, ceetm_node, child; + pcell_t range[2]; + enum qm_dc_portal dcp_portal; + struct qm_ceetm_sp *sp; + struct qm_ceetm_lni *lni; + int ret, i, found; + + mtx_init(&ceetm_alloc_lock, "ceetm_alloc", NULL, MTX_DEF); + qman_num_ceetms = 0; + + /* Find the qman node */ + qman_node = ofw_bus_get_node(qman_sc->sc_dev); + if (qman_node <= 0) + return (0); /* No DT node — CEETM not available */ + + /* + * Get QMan clock frequency. + * Try "clock-frequency" on qman node first, then fall back to + * the soc node's "bus-frequency", then default to 400MHz. + */ + qman_clk = 0; + if (OF_getencprop(qman_node, "clock-frequency", + (void *)&qman_clk, sizeof(qman_clk)) <= 0) { + phandle_t soc_node; + + soc_node = OF_finddevice("/soc"); + if (soc_node > 0) + OF_getencprop(soc_node, "bus-frequency", + (void *)&qman_clk, sizeof(qman_clk)); + } + if (qman_clk == 0) { + printf("ceetm: no clock-frequency, defaulting to 400MHz\n"); + qman_clk = 400000000; + } + /* + * Search for fsl,qman-ceetm child nodes under the qman-portals + * parent, or directly under qman. + * The DT layout varies — check all children of the qman portals + * node and the qman node itself. + */ + found = 0; + + /* Search children of qman node */ + for (child = OF_child(qman_node); child > 0; child = OF_peer(child)) { + if (!ofw_bus_node_is_compatible(child, "fsl,qman-ceetm")) + continue; + if (!ofw_bus_node_status_okay(child)) + continue; + + /* Parse LFQID range — extract DCP portal index */ + if (OF_getencprop(child, "fsl,ceetm-lfqid-range", + (void *)range, sizeof(range)) != sizeof(range)) { + printf("ceetm: missing fsl,ceetm-lfqid-range\n"); + continue; + } + + dcp_portal = (range[0] & 0x0F0000) >> 16; + if (dcp_portal >= QMAN_CEETM_MAX) { + printf("ceetm: DCP portal %d out of range\n", + dcp_portal); + continue; + } + + /* Seed LFQID allocator */ + ceetm_pool_seed(&ceetm_lfqid_pool[dcp_portal], + range[0], range[1]); + + /* Initialize CEETM instance */ + qman_ceetms[dcp_portal].idx = dcp_portal; + TAILQ_INIT(&qman_ceetms[dcp_portal].sub_portals); + TAILQ_INIT(&qman_ceetms[dcp_portal].lnis); + + /* Parse SP range */ + if (OF_getencprop(child, "fsl,ceetm-sp-range", + (void *)range, sizeof(range)) != sizeof(range)) { + printf("ceetm: missing fsl,ceetm-sp-range\n"); + continue; + } + qman_ceetms[dcp_portal].sp_range[0] = range[0]; + qman_ceetms[dcp_portal].sp_range[1] = range[1]; + + for (i = 0; i < (int)range[1]; i++) { + sp = malloc(sizeof(*sp), M_NETCOMMSW, + M_WAITOK | M_ZERO); + sp->idx = range[0] + i; + sp->dcp_idx = dcp_portal; + TAILQ_INSERT_TAIL( + &qman_ceetms[dcp_portal].sub_portals, + sp, entries); + } + + /* Parse LNI range */ + if (OF_getencprop(child, "fsl,ceetm-lni-range", + (void *)range, sizeof(range)) != sizeof(range)) { + printf("ceetm: missing fsl,ceetm-lni-range\n"); + continue; + } + qman_ceetms[dcp_portal].lni_range[0] = range[0]; + qman_ceetms[dcp_portal].lni_range[1] = range[1]; + + for (i = 0; i < (int)range[1]; i++) { + lni = malloc(sizeof(*lni), M_NETCOMMSW, + M_WAITOK | M_ZERO); + lni->idx = range[0] + i; + lni->dcp_idx = dcp_portal; + TAILQ_INIT(&lni->channels); + TAILQ_INSERT_TAIL( + &qman_ceetms[dcp_portal].lnis, + lni, entries); + } + + /* Parse channel range */ + if (OF_getencprop(child, "fsl,ceetm-channel-range", + (void *)range, sizeof(range)) != sizeof(range)) { + printf("ceetm: missing fsl,ceetm-channel-range\n"); + continue; + } + ceetm_pool_seed(&ceetm_channel_pool[dcp_portal], + range[0], range[1]); + + /* Set prescaler */ + ret = qman_ceetm_set_prescaler(dcp_portal); + if (ret) { + printf("ceetm: failed to set prescaler for DCP%d\n", + dcp_portal); + continue; + } + + if (dcp_portal >= qman_num_ceetms) + qman_num_ceetms = dcp_portal + 1; + found++; + + printf("ceetm: DCP%d initialized: %d SPs, %d LNIs, " + "clk=%uMHz\n", + dcp_portal, + qman_ceetms[dcp_portal].sp_range[1], + qman_ceetms[dcp_portal].lni_range[1], + qman_clk / 1000000); + } + + if (found == 0) { + /* Also search under qman-portals parent node */ + ceetm_node = OF_finddevice("/soc/fsl,dpaa/fsl,qman-portals"); + if (ceetm_node > 0) { + for (child = OF_child(ceetm_node); child > 0; + child = OF_peer(child)) { + if (ofw_bus_node_is_compatible(child, + "fsl,qman-ceetm")) + printf("ceetm: found CEETM under " + "qman-portals, not yet " + "supported — add to qman " + "node instead\n"); + } + } + } + + if (qman_num_ceetms > 0) + printf("ceetm: %d CEETM instance%s available\n", + qman_num_ceetms, + qman_num_ceetms > 1 ? "s" : ""); + + return (0); +} diff --git a/sys/dev/dpaa/qman_ceetm.h b/sys/dev/dpaa/qman_ceetm.h new file mode 100644 index 000000000000..42e6fe839d5b --- /dev/null +++ b/sys/dev/dpaa/qman_ceetm.h @@ -0,0 +1,719 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Mono Technologies Inc. + * + * QMan CEETM (Channel Egress Enhancement and Traffic Management) driver + * for FreeBSD. Ported from the Linux NXP QBMan SDK driver. + * + * CEETM provides hierarchical egress QoS: + * Sub-portals → LNIs → Channels → Class Queues + * with per-level token-bucket shapers and WBFQ scheduling. + * + * Original Linux source: + * drivers/staging/fsl_qbman/qman_high.c (API implementation) + * include/linux/fsl_qman.h (types, MC structures) + * drivers/staging/fsl_qbman/qman_driver.c (DT init) + * drivers/staging/fsl_qbman/qman_config.c (CCSR registers) + */ + +#ifndef _QMAN_CEETM_H +#define _QMAN_CEETM_H + +#include +#include + +/* + * Forward declarations — NCSW types used by qman_ceetm.c but not + * needed by external callers. + */ +struct qm_portal; + +/* ================================================================ + * CEETM Management Command Verb Codes (QMan RM section 1.5.9.7) + * + * These are sent via the same portal MC interface as FQ/CGR commands + * (qm_mc_start / qm_mc_commit / qm_mc_result). + * ================================================================ */ + +#define QM_CEETM_VERB_LFQMT_CONFIG 0x70 +#define QM_CEETM_VERB_LFQMT_QUERY 0x71 +#define QM_CEETM_VERB_CQ_CONFIG 0x72 +#define QM_CEETM_VERB_CQ_QUERY 0x73 +#define QM_CEETM_VERB_DCT_CONFIG 0x74 +#define QM_CEETM_VERB_DCT_QUERY 0x75 +#define QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG 0x76 +#define QM_CEETM_VERB_CLASS_SCHEDULER_QUERY 0x77 +#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG 0x78 +#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY 0x79 +#define QM_CEETM_VERB_CCGR_CONFIG 0x7A +#define QM_CEETM_VERB_CCGR_QUERY 0x7B +#define QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD 0x7C +#define QM_CEETM_VERB_STATISTICS_QUERY_WRITE 0x7D + +/* MAPPING_SHAPER_TCFC sub-command types (upper 4 bits of cid field) */ +#define CEETM_COMMAND_CHANNEL_MAPPING (0 << 12) +#define CEETM_COMMAND_SP_MAPPING (1 << 12) +#define CEETM_COMMAND_CHANNEL_SHAPER (2 << 12) +#define CEETM_COMMAND_LNI_SHAPER (3 << 12) +#define CEETM_COMMAND_TCFC (4 << 12) + +/* CCGR sub-command types (upper 2 bits of ccgrid field) */ +#define CEETM_CCGRID_MASK 0x01FF +#define CEETM_CCGR_CM_CONFIGURE (0 << 14) +#define CEETM_CCGR_DN_CONFIGURE (1 << 14) +#define CEETM_CCGR_TEST_WRITE (2 << 14) +#define CEETM_CCGR_CM_QUERY (0 << 14) +#define CEETM_CCGR_DN_QUERY (1 << 14) +#define CEETM_CCGR_DN_QUERY_FLUSH (2 << 14) +#define CEETM_QUERY_CONGESTION_STATE (3 << 14) + +/* Statistics query/write command types */ +#define CEETM_QUERY_DEQUEUE_STATISTICS 0x00 +#define CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS 0x01 +#define CEETM_WRITE_DEQUEUE_STATISTICS 0x02 +#define CEETM_QUERY_REJECT_STATISTICS 0x03 +#define CEETM_QUERY_REJECT_CLEAR_STATISTICS 0x04 +#define CEETM_WRITE_REJECT_STATISTICS 0x05 + +/* MC result codes */ +#define QM_MCR_RESULT_OK 0xf0 +#define QM_MCR_RESULT_ERR_BADCOMMAND 0xff + +/* DCP_CFG CEETM enable bit for a sub-portal */ +#define QM_SP_ENABLE_CEETM(sp) (0x80000000 >> (sp)) + +/* ================================================================ + * CEETM MC Command Structures (MCC) + * + * Each is exactly 63 bytes (64 - 1 verb byte). These overlay the + * generic qm_mc_command union returned by qm_mc_start(). All + * multi-byte fields are big-endian on the wire; callers use + * WRITE_UINT16/WRITE_UINT32 macros. + * + * Field layout matches Linux include/linux/fsl_qman.h verbatim. + * ================================================================ */ + +struct qm_mcc_ceetm_lfqmt_config { + uint8_t __reserved1[3]; + uint8_t lfqid_hi; /* lfqid[23:16] */ + uint16_t lfqid_lo; /* lfqid[15:0] */ + uint8_t __reserved2[2]; + uint16_t cqid; + uint8_t __reserved3[2]; + uint16_t dctidx; + uint8_t __reserved4[48]; +} __packed; + +struct qm_mcc_ceetm_lfqmt_query { + uint8_t __reserved1[3]; + uint8_t lfqid_hi; + uint16_t lfqid_lo; + uint8_t __reserved2[56]; +} __packed; + +struct qm_mcc_ceetm_cq_config { + uint16_t cqid; + uint8_t dcpid; + uint8_t __reserved1; + uint16_t ccgid; + uint8_t __reserved2[56]; +} __packed; + +struct qm_mcc_ceetm_cq_query { + uint16_t cqid; + uint8_t dcpid; + uint8_t __reserved1[59]; +} __packed; + +struct qm_mcc_ceetm_dct_config { + uint16_t dctidx; + uint8_t dcpid; + uint8_t __reserved1[15]; + uint32_t context_b; + uint64_t context_a; + uint8_t __reserved2[32]; +} __packed; + +struct qm_mcc_ceetm_dct_query { + uint16_t dctidx; + uint8_t dcpid; + uint8_t __reserved1[59]; +} __packed; + +/* + * Class scheduler config — GPC byte layout (big-endian on wire): + * bit 7: reserved + * bit 6: combine_flag + * bits 5-3: prio_b + * bits 2-0: prio_a + */ +#define CEETM_GPC_COMBINE_FLAG 0x40 +#define CEETM_GPC_PRIO_B_SHIFT 3 +#define CEETM_GPC_PRIO_B_MASK 0x38 +#define CEETM_GPC_PRIO_A_MASK 0x07 + +struct qm_mcc_ceetm_class_scheduler_config { + uint16_t cqcid; /* channel CQ channel id */ + uint8_t dcpid; + uint8_t __reserved1[6]; + uint8_t gpc; /* group priority control */ + uint16_t crem; /* CR eligibility mask */ + uint16_t erem; /* ER eligibility mask */ + uint8_t w[8]; /* weight codes */ + uint8_t __reserved2[40]; +} __packed; + +struct qm_mcc_ceetm_class_scheduler_query { + uint16_t cqcid; + uint8_t dcpid; + uint8_t __reserved1[59]; +} __packed; + +/* + * MAPPING_SHAPER_TCFC config — multiplexed command. + * The upper 4 bits of cid select the sub-command type. + */ +struct qm_mcc_ceetm_mapping_shaper_tcfc_config { + uint16_t cid; /* sub-command type | resource index */ + uint8_t dcpid; + union { + struct { + uint8_t map_flags; /* bit7=shaped, bits2-0=lni_id */ + uint8_t __reserved[58]; + } __packed channel_mapping; + struct { + uint8_t map_flags; /* bits2-0=lni_id */ + uint8_t __reserved[58]; + } __packed sp_mapping; + struct { + uint8_t cpl_oal; /* bit7=coupled, bits4-0=oal */ + uint8_t crtcr_hi; /* crtcr[23:16] */ + uint16_t crtcr_lo; /* crtcr[15:0] */ + uint8_t ertcr_hi; /* ertcr[23:16] */ + uint16_t ertcr_lo; /* ertcr[15:0] */ + uint16_t crtbl; + uint16_t ertbl; + uint8_t mps; + uint8_t __reserved[47]; + } __packed shaper_config; + struct { + uint8_t __reserved1[11]; + uint64_t lnitcfcc; + uint8_t __reserved2[40]; + } __packed tcfc_config; + }; +} __packed; + +struct qm_mcc_ceetm_mapping_shaper_tcfc_query { + uint16_t cid; + uint8_t dcpid; + uint8_t __reserved[59]; +} __packed; + +/* + * CCGR config — Congestion Manager or Dequeue Notification config. + * We only use CM config (tail-drop threshold for class queues). + */ +struct qm_mcc_ceetm_ccgr_config { + uint16_t ccgrid; /* sub-command | ccg index */ + uint8_t dcpid; + uint8_t __reserved1; + uint16_t we_mask; + /* CM config fields (when ccgrid sub-command = CM_CONFIGURE) */ + uint8_t ctl; /* control byte */ + uint8_t cdv; + uint16_t cscn_tupd; + uint8_t oal; + uint8_t __reserved2; + uint16_t cs_thres; /* qm_cgr_cs_thres format */ + uint16_t cs_thres_x; + uint16_t td_thres; + uint32_t wr_parm_g; /* qm_cgr_wr_parm format */ + uint32_t wr_parm_y; + uint32_t wr_parm_r; + uint8_t __reserved3[32]; +} __packed; + +struct qm_mcc_ceetm_ccgr_query { + uint16_t ccgrid; + uint8_t dcpid; + uint8_t __reserved[59]; +} __packed; + +struct qm_mcc_ceetm_cq_peek_pop_xsfdrread { + uint16_t cqid; + uint8_t dcpid; + uint8_t ct; + uint16_t xsfdr; + uint8_t __reserved[56]; +} __packed; + +struct qm_mcc_ceetm_statistics_query_write { + uint16_t cid; + uint8_t dcpid; + uint8_t ct; + uint8_t __reserved1[13]; + uint8_t frm_cnt[5]; /* 40-bit frame count */ + uint8_t __reserved2[2]; + uint8_t byte_cnt[6]; /* 48-bit byte count */ + uint8_t __reserved3[32]; +} __packed; + +/* ================================================================ + * CEETM MC Result Structures (MCR) + * + * Each is exactly 64 bytes. First two bytes are verb + result. + * These overlay the generic qm_mc_result returned by qm_mc_result(). + * ================================================================ */ + +struct qm_mcr_ceetm_generic { + uint8_t verb; + uint8_t result; + uint8_t __reserved[62]; +} __packed; + +/* LFQMT query result */ +struct qm_mcr_ceetm_lfqmt_query { + uint8_t verb; + uint8_t result; + uint8_t __reserved1[8]; + uint16_t cqid; + uint8_t __reserved2[2]; + uint16_t dctidx; + uint8_t __reserved3[2]; + uint16_t ccgid; + uint8_t __reserved4[44]; +} __packed; + +/* CQ query result */ +struct qm_mcr_ceetm_cq_query { + uint8_t verb; + uint8_t result; + uint8_t __reserved1[4]; + uint16_t ccgid; + uint16_t state; + uint8_t pfdr_hptr_hi; + uint16_t pfdr_hptr_lo; + uint8_t pfdr_tptr_hi; + uint16_t pfdr_tptr_lo; + uint16_t od1_xsfdr; + uint16_t od2_xsfdr; + uint16_t od3_xsfdr; + uint16_t od4_xsfdr; + uint16_t od5_xsfdr; + uint16_t od6_xsfdr; + uint16_t ra1_xsfdr; + uint16_t ra2_xsfdr; + uint8_t __reserved2; + uint8_t frm_cnt_hi; + uint16_t frm_cnt_lo; + uint8_t __reserved3[28]; +} __packed; + +/* DCT query result */ +struct qm_mcr_ceetm_dct_query { + uint8_t verb; + uint8_t result; + uint8_t __reserved1[18]; + uint32_t context_b; + uint64_t context_a; + uint8_t __reserved2[32]; +} __packed; + +/* Class scheduler query result */ +struct qm_mcr_ceetm_class_scheduler_query { + uint8_t verb; + uint8_t result; + uint8_t __reserved1[9]; + uint8_t gpc; + uint16_t crem; + uint16_t erem; + uint8_t w[8]; + uint8_t __reserved2[5]; + uint8_t wbfslist_hi; + uint16_t wbfslist_lo; + uint32_t d8; + uint32_t d9; + uint32_t d10; + uint32_t d11; + uint32_t d12; + uint32_t d13; + uint32_t d14; + uint32_t d15; +} __packed; + +/* MAPPING_SHAPER_TCFC query result */ +struct qm_mcr_ceetm_mapping_shaper_tcfc_query { + uint8_t verb; + uint8_t result; + uint16_t cid; + uint8_t __reserved1; + union { + struct { + uint8_t map_flags; + uint8_t __reserved[58]; + } __packed channel_mapping_query; + struct { + uint8_t map_flags; + uint8_t __reserved[58]; + } __packed sp_mapping_query; + struct { + uint8_t cpl_oal; + uint8_t crtcr_hi; + uint16_t crtcr_lo; + uint8_t ertcr_hi; + uint16_t ertcr_lo; + uint16_t crtbl; + uint16_t ertbl; + uint8_t mps; + uint8_t __reserved1[15]; + uint32_t crat; + uint32_t erat; + uint8_t __reserved2[24]; + } __packed shaper_query; + struct { + uint8_t __reserved1[11]; + uint64_t lnitcfcc; + uint8_t __reserved2[40]; + } __packed tcfc_query; + }; +} __packed; + +/* CCGR config result (test-write variant has extra fields) */ +struct qm_mcr_ceetm_ccgr_config { + uint8_t verb; + uint8_t result; + uint8_t __reserved1[62]; +} __packed; + +/* CCGR query result (CM query variant) */ +struct qm_mcr_ceetm_ccgr_query { + uint8_t verb; + uint8_t result; + uint8_t __reserved1[6]; + /* CM query fields */ + uint8_t ctl; + uint8_t cdv; + uint8_t __reserved2[2]; + uint8_t oal; + uint8_t __reserved3; + uint16_t cs_thres; + uint16_t cs_thres_x; + uint16_t td_thres; + uint32_t wr_parm_g; + uint32_t wr_parm_y; + uint32_t wr_parm_r; + uint16_t cscn_targ_dcp; + uint8_t dcp_lsn; + uint8_t i_cnt[5]; /* 40-bit */ + uint8_t __reserved4[3]; + uint8_t a_cnt[5]; /* 40-bit */ + uint32_t cscn_targ_swp[4]; +} __packed; + +/* CQ peek/pop result */ +struct qm_mcr_ceetm_cq_peek_pop_xsfdrread { + uint8_t verb; + uint8_t result; + uint8_t stat; + uint8_t __reserved1[61]; +} __packed; + +/* Statistics query result */ +struct qm_mcr_ceetm_statistics_query { + uint8_t verb; + uint8_t result; + uint8_t __reserved1[17]; + uint8_t frm_cnt[5]; /* 40-bit frame count */ + uint8_t __reserved2[2]; + uint8_t byte_cnt[6]; /* 48-bit byte count */ + uint8_t __reserved3[32]; +} __packed; + +/* ================================================================ + * CEETM High-Level Data Structures + * + * These represent the CEETM resource hierarchy. Ported from + * Linux include/linux/fsl_qman.h, adapted to FreeBSD conventions. + * ================================================================ */ + +/* Maximum CEETM instances (one per FMan / DCP portal) */ +#define QMAN_CEETM_MAX 2 + +/* Token rate (for shapers) */ +struct qm_ceetm_rate { + uint32_t whole; /* integer part (0..2047) */ + uint32_t fraction; /* fractional part (0..8191) */ +}; + +/* WBFS weight code */ +struct qm_ceetm_weight_code { + uint8_t y; /* 0..31 */ + uint8_t x; /* 0..7 */ + /* effective weight = 2^(x+6) / (64 - y) */ +}; + +/* DCP portal enum — already defined in NCSW fsl_qman.h for kernel context */ +#ifndef __FSL_QMAN_H +enum qm_dc_portal { + qm_dc_portal_fman0 = 0, + qm_dc_portal_fman1 = 1, +}; +#endif + +/* Sub-portal */ +struct qm_ceetm_sp { + TAILQ_ENTRY(qm_ceetm_sp) entries; + unsigned int idx; + unsigned int dcp_idx; + int is_claimed; + struct qm_ceetm_lni *lni; +}; + +/* Logical Network Interface */ +struct qm_ceetm_lni { + TAILQ_ENTRY(qm_ceetm_lni) entries; + unsigned int idx; + unsigned int dcp_idx; + int is_claimed; + struct qm_ceetm_sp *sp; + TAILQ_HEAD(, qm_ceetm_channel) channels; + int shaper_enable; + int shaper_couple; + int oal; + struct qm_ceetm_rate cr_token_rate; + struct qm_ceetm_rate er_token_rate; + uint16_t cr_token_bucket_limit; + uint16_t er_token_bucket_limit; +}; + +/* CQ Channel */ +struct qm_ceetm_channel { + TAILQ_ENTRY(qm_ceetm_channel) entries; + struct qm_ceetm_lni *lni; /* parent LNI */ + unsigned int idx; + unsigned int lni_idx; + unsigned int dcp_idx; + TAILQ_HEAD(, qm_ceetm_cq) class_queues; + TAILQ_HEAD(, qm_ceetm_ccg) ccgs; + uint8_t shaper_enable; + uint8_t shaper_couple; + struct qm_ceetm_rate cr_token_rate; + struct qm_ceetm_rate er_token_rate; + uint16_t cr_token_bucket_limit; + uint16_t er_token_bucket_limit; +}; + +/* Class Queue */ +struct qm_ceetm_cq { + TAILQ_ENTRY(qm_ceetm_cq) entries; + struct qm_ceetm_channel *parent; + struct qm_ceetm_ccg *ccg; + unsigned int idx; /* 0-7 individual, 8-15 grouped */ + int is_claimed; + TAILQ_HEAD(, qm_ceetm_lfq) bound_lfqids; +}; + +/* Logical Frame Queue */ +struct qm_ceetm_lfq { + TAILQ_ENTRY(qm_ceetm_lfq) entries; + struct qm_ceetm_cq *parent; + unsigned int idx; /* LFQID */ + unsigned int dctidx; /* dequeue context table index */ + uint64_t context_a; + uint32_t context_b; +}; + +/* Class Congestion Group */ +struct qm_ceetm_ccg { + TAILQ_ENTRY(qm_ceetm_ccg) entries; + struct qm_ceetm_channel *parent; + unsigned int idx; /* 0-15 */ +}; + +/* Top-level CEETM instance (one per DCP/FMan) */ +struct qm_ceetm { + unsigned int idx; /* DCP portal index */ + TAILQ_HEAD(, qm_ceetm_sp) sub_portals; + TAILQ_HEAD(, qm_ceetm_lni) lnis; + unsigned int sp_range[2]; /* [start, count] */ + unsigned int lni_range[2]; +}; + +/* ================================================================ + * CEETM CCSR Register Offsets (relative to QMan CCSR base) + * ================================================================ */ + +#define QM_REG_DCP_CFG(n) (0x0300 + ((n) * 0x10)) +#define QM_REG_CEETM_CFG_IDX 0x0900 +#define QM_REG_CEETM_CFG_PRES 0x0904 +#define QM_REG_CEETM_XSFDR_IN_USE 0x0908 + +/* ================================================================ + * CEETM CCGR WE (Write Enable) Mask Bits + * ================================================================ */ + +#define QM_CEETM_CCGR_WE_CSCN_EN 0x4000 +#define QM_CEETM_CCGR_WE_TD_EN 0x2000 +#define QM_CEETM_CCGR_WE_TD_MODE 0x1000 +#define QM_CEETM_CCGR_WE_TD_THRES 0x0800 +#define QM_CEETM_CCGR_WE_OAL 0x0400 +#define QM_CEETM_CCGR_WE_CDV 0x0200 +#define QM_CEETM_CCGR_WE_CSCN_TUPD 0x0100 +#define QM_CEETM_CCGR_WE_CS_THRES 0x0080 +#define QM_CEETM_CCGR_WE_CS_THRES_X 0x0040 +#define QM_CEETM_CCGR_WE_MODE 0x0020 +#define QM_CEETM_CCGR_WE_WR_PARM_G 0x0010 +#define QM_CEETM_CCGR_WE_WR_PARM_Y 0x0008 +#define QM_CEETM_CCGR_WE_WR_PARM_R 0x0004 +#define QM_CEETM_CCGR_WE_DN 0x0002 + +/* CCGR CTL byte bit definitions */ +#define QM_CEETM_CCGR_CTL_WR_EN_G 0x40 +#define QM_CEETM_CCGR_CTL_WR_EN_Y 0x20 +#define QM_CEETM_CCGR_CTL_WR_EN_R 0x10 +#define QM_CEETM_CCGR_CTL_TD_EN 0x08 +#define QM_CEETM_CCGR_CTL_TD_MODE 0x04 +#define QM_CEETM_CCGR_CTL_CSCN_EN 0x02 +#define QM_CEETM_CCGR_CTL_MODE 0x01 + +/* ================================================================ + * Public API — implemented in qman_ceetm.c + * ================================================================ */ + +/* Initialization (called from qman_attach if DT node present) */ +int qman_ceetm_init(void); + +/* Global state */ +extern uint8_t qman_num_ceetms; +extern struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX]; + +/* Pure math */ +int qman_ceetm_bps2tokenrate(uint64_t bps, + struct qm_ceetm_rate *token_rate, int rounding); +int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, + uint64_t *bps, int rounding); +int qman_ceetm_ratio2wbfs(uint32_t numerator, uint32_t denominator, + struct qm_ceetm_weight_code *weight_code, int rounding); +int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code, + uint32_t *numerator, uint32_t *denominator); + +/* Sub-portal management */ +int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, + enum qm_dc_portal dcp_idx, unsigned int sp_idx); +int qman_ceetm_sp_release(struct qm_ceetm_sp *sp); +int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, + struct qm_ceetm_lni *lni); +int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, unsigned int *lni_idx); + +/* LNI management */ +int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, + enum qm_dc_portal dcp_idx, unsigned int lni_idx); +int qman_ceetm_lni_release(struct qm_ceetm_lni *lni); +int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, + int coupled, int oal); +int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni); +int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni); +int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni, + const struct qm_ceetm_rate *cr, uint16_t cr_limit); +int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni, + const struct qm_ceetm_rate *er, uint16_t er_limit); +int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni, + struct qm_ceetm_rate *cr, uint16_t *cr_limit); +int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni, + struct qm_ceetm_rate *er, uint16_t *er_limit); + +/* Channel management */ +int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel, + struct qm_ceetm_lni *lni); +int qman_ceetm_channel_release(struct qm_ceetm_channel *channel); +int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel, + int coupled); +int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel); +int qman_ceetm_channel_is_shaper_enabled( + struct qm_ceetm_channel *channel); +int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel, + const struct qm_ceetm_rate *cr, uint16_t cr_limit); +int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel, + const struct qm_ceetm_rate *er, uint16_t er_limit); +int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel, + struct qm_ceetm_rate *cr, uint16_t *cr_limit); +int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel, + struct qm_ceetm_rate *er, uint16_t *er_limit); +int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, + int group_b, unsigned int prio_a, unsigned int prio_b); +int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, + int *group_b, unsigned int *prio_a, unsigned int *prio_b); +int qman_ceetm_channel_set_group_cr_eligibility( + struct qm_ceetm_channel *channel, int group_b, int cre); +int qman_ceetm_channel_set_group_er_eligibility( + struct qm_ceetm_channel *channel, int group_b, int ere); +int qman_ceetm_channel_set_cq_cr_eligibility( + struct qm_ceetm_channel *channel, unsigned int idx, int cre); +int qman_ceetm_channel_set_cq_er_eligibility( + struct qm_ceetm_channel *channel, unsigned int idx, int ere); + +/* Class queue management */ +int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq, + struct qm_ceetm_channel *channel, unsigned int idx, + struct qm_ceetm_ccg *ccg); +int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq, + struct qm_ceetm_channel *channel, unsigned int idx, + struct qm_ceetm_ccg *ccg); +int qman_ceetm_cq_release(struct qm_ceetm_cq *cq); +int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq, + struct qm_ceetm_weight_code *weight_code); +int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq, + struct qm_ceetm_weight_code *weight_code); + +/* LFQ management */ +int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq, + struct qm_ceetm_cq *cq); +int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq); +int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, + uint64_t context_a, uint32_t context_b); +int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, + uint64_t *context_a, uint32_t *context_b); + +/* + * QMan FQ for CEETM egress — minimal struct matching Linux fsl_qman.h. + * In kernel context, NCSW qm.h provides the full struct qman_fq. + * In module context, dpaa_eth.h provides the minimal struct. + */ +#if !defined(__QM_H) && !defined(_QMAN_FQ_DEFINED) +#define _QMAN_FQ_DEFINED +struct qman_fq { + uint32_t fqid; + uint32_t flags; +}; +#endif +int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, + struct qman_fq *fq); + +/* CCG management */ +int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg, + struct qm_ceetm_channel *channel, unsigned int idx, + void (*cb)(void *, int), void *cb_ctx); +int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg); +int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, + uint16_t we_mask, struct qm_mcc_ceetm_ccgr_config *opts); +int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, + uint32_t flags, uint64_t *frame_count, uint64_t *byte_count); + +/* Statistics/query */ +int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid, + struct qm_mcr_ceetm_cq_query *cq_query); +int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, + uint32_t flags, uint64_t *frame_count, uint64_t *byte_count); + +/* CCSR register access */ +int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, + uint16_t sub_portal); +int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, + uint16_t sub_portal); +int qman_ceetm_configure_mapping_shaper_tcfc( + struct qm_mcc_ceetm_mapping_shaper_tcfc_config *opts); +int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num); + +#endif /* _QMAN_CEETM_H */ diff --git a/sys/dev/dpaa/qman_fdt.c b/sys/dev/dpaa/qman_fdt.c index fac845c8f2b5..6c6263258e6a 100644 --- a/sys/dev/dpaa/qman_fdt.c +++ b/sys/dev/dpaa/qman_fdt.c @@ -103,7 +103,10 @@ static driver_t qm_portals_driver = { }; EARLY_DRIVER_MODULE(qman_portals, ofwbus, qm_portals_driver, 0, 0, - BUS_PASS_BUS); + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LAST); +/* On ARM64, portal bus nodes are children of soc (simplebus), not ofwbus */ +EARLY_DRIVER_MODULE(qman_portals, simplebus, qm_portals_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LAST); static void get_addr_props(phandle_t node, uint32_t *addrp, uint32_t *sizep) @@ -147,7 +150,7 @@ qman_portal_find_cpu(int cpu) return (-1); for (node = OF_child(node); node != 0; node = OF_peer(node)) { - if (OF_getprop(node, "reg", ®, sizeof(reg)) <= 0) + if (OF_getencprop(node, "reg", ®, sizeof(reg)) <= 0) continue; if (reg == cpu) return (node); @@ -212,14 +215,16 @@ qman_portals_fdt_attach(device_t dev) continue; } /* Checkout related cpu */ - if (OF_getprop(child, "cpu-handle", (void *)&cpu, + if (OF_getencprop(child, "cpu-handle", (void *)&cpu, sizeof(cpu)) <= 0) { - cpu = qman_portal_find_cpu(cpus); - if (cpu <= 0) + /* No cpu-handle (ARM64): find CPU node directly */ + cpu_node = qman_portal_find_cpu(cpus); + if (cpu_node <= 0) continue; + } else { + /* cpu-handle is an ihandle (PowerPC) */ + cpu_node = OF_instance_to_package(cpu); } - /* Acquire cpu number */ - cpu_node = OF_instance_to_package(cpu); if (OF_getencprop(cpu_node, "reg", &cpu_num, sizeof(cpu_num)) <= 0) { device_printf(dev, "Could not retrieve CPU number.\n"); return (ENXIO); diff --git a/sys/dev/dpaa/qman_portals.c b/sys/dev/dpaa/qman_portals.c index 76d89784ab5b..d645d4a76855 100644 --- a/sys/dev/dpaa/qman_portals.c +++ b/sys/dev/dpaa/qman_portals.c @@ -36,15 +36,19 @@ #include #include #include +#include #include +#include +#include +#ifdef __powerpc__ #include +#include +#endif #include #include -#include - #include "qman.h" #include "portals.h" @@ -60,15 +64,16 @@ struct dpaa_portals_softc *qp_sc; int qman_portals_attach(device_t dev) { - struct dpaa_portals_softc *sc; + qp_sc = device_get_softc(dev); - sc = qp_sc = device_get_softc(dev); - - /* Map bman portal to physical address space */ +#ifdef __powerpc__ + struct dpaa_portals_softc *sc = qp_sc; + /* Map qman portal to physical address space (PowerPC LAW) */ if (law_enable(OCP85XX_TGTIF_QMAN, sc->sc_dp_pa, sc->sc_dp_size)) { qman_portals_detach(dev); return (ENXIO); } +#endif /* Set portal properties for XX_VirtToPhys() */ XX_PortalSetInfo(dev); @@ -128,6 +133,19 @@ qman_portal_setup(struct qman_softc *qsc) sc = qp_sc; +#ifdef __aarch64__ + /* + * On ARM64, all portals are pre-initialized with NAPI-style + * interrupt handling via qman_portal_init_cpu() in qman_attach(). + * Just return the portal for the current CPU. + */ + sched_pin(); + cpu = PCPU_GET(cpuid); + portal = sc->sc_dp[cpu].dp_ph; + sched_unpin(); + return (portal); +#endif + sched_pin(); portal = NULL; cpu = PCPU_GET(cpuid); @@ -150,9 +168,14 @@ qman_portal_setup(struct qman_softc *qsc) /* Map portal registers */ dpaa_portal_map_registers(sc); + + /* Bail if portal mapping failed (e.g. no portal allocated for this CPU) */ + if (sc->sc_dp[cpu].dp_ce_va == 0 || sc->sc_dp[cpu].dp_ci_va == 0) + goto err; + /* Configure and initialize portal */ - qpp.ceBaseAddress = rman_get_bushandle(sc->sc_rres[0]); - qpp.ciBaseAddress = rman_get_bushandle(sc->sc_rres[1]); + qpp.ceBaseAddress = sc->sc_dp[cpu].dp_ce_va; + qpp.ciBaseAddress = sc->sc_dp[cpu].dp_ci_va; qpp.h_Qm = qsc->sc_qh; qpp.swPortalId = cpu; qpp.irq = (uintptr_t)sc->sc_dp[cpu].dp_ires; @@ -186,3 +209,250 @@ qman_portal_setup(struct qman_softc *qsc) return (NULL); } + +#ifdef __aarch64__ + +#include +#include +#include +#include +#include +#include + +/* Per-portal NAPI state */ +static struct { + struct task task; + struct taskqueue *tq; + struct mbufq rxq; /* deferred RX mbufs */ +} qman_napi[MAXCPU]; + +/* Declared in xx_arm64.c */ +extern void XX_ConfigPortalNapi(uintptr_t irq, struct taskqueue *tq, + struct task *task); +extern void XX_SetPortalNapiHandle(uintptr_t irq, t_Handle portal); + +/* + * Enqueue an mbuf for deferred delivery after QM_PORTAL_Poll returns. + * Called from the DQRR RX callback (inside NCSW_PLOCK) to avoid + * re-entering the portal via if_input -> TCP -> TX -> QM_FQR_Enqueue. + */ +void +qman_rx_defer(struct mbuf *m) +{ + int cpu = PCPU_GET(cpuid); + + if (mbufq_enqueue(&qman_napi[cpu].rxq, m) != 0) { + static volatile uint32_t defer_drop_cnt; + uint32_t n = atomic_fetchadd_32(&defer_drop_cnt, 1) + 1; + if (n <= 5) + printf("qman: rx_defer queue full on CPU %d (#%u)," + " dropping\n", cpu, n); + m_freem(m); + } +} + +/* + * NAPI-style poll task for QMan portals (FreeBSD equivalent of Linux + * dpaa_eth_poll). Runs in a per-CPU taskqueue thread at PI_NET, + * scheduled by the FILTER interrupt handler via XX_PortalFilter. + * + * Drains the entire DQRR (max 16 entries) and MR in one shot via + * QM_PORTAL_Poll, then delivers deferred RX mbufs outside NCSW_PLOCK + * to prevent nested portal enqueue from the TX path. + */ +static void +qman_portal_poll_task(void *arg, int pending) +{ + t_Handle portal = arg; + struct mbuf *m; + int cpu = PCPU_GET(cpuid); + + QM_PORTAL_Poll(portal, e_QM_PORTAL_POLL_SOURCE_BOTH); + QM_PORTAL_Uninhibit(portal); + + /* + * Deliver deferred mbufs outside NCSW_PLOCK. + * M_PROTO1-tagged mbufs are WiFi TX (CDX→WiFi from dpaa_wifi) — + * deliver via if_transmit instead of if_input. + */ + while ((m = mbufq_dequeue(&qman_napi[cpu].rxq)) != NULL) { + if (__predict_false(m->m_flags & M_PROTO1)) { + m->m_flags &= ~M_PROTO1; + if_transmit(m->m_pkthdr.rcvif, m); + } else { + if_input(m->m_pkthdr.rcvif, m); + } + } +} + +/* + * qman_portal_quiesce — Drain all per-CPU portal poll taskqueues. + * + * Waits for any in-flight QM_PORTAL_Poll() call to complete on every + * CPU. Must be called after retiring FQRs to ensure no portal poll + * task references a stale DQRR callback from a module about to be + * unloaded. + */ +void +qman_portal_quiesce(void) +{ + int cpu; + + for (cpu = 0; cpu < mp_ncpus; cpu++) { + if (qman_napi[cpu].tq != NULL) + taskqueue_drain(qman_napi[cpu].tq, + &qman_napi[cpu].task); + } +} + +/* + * Initialize a QMan portal for a specific CPU with NAPI-style + * interrupt handling. Like Linux's qman_create_affine_portal + + * dpaa_eth_add_channel: + * + * 1. Create a per-CPU taskqueue pinned to the portal's CPU + * 2. Configure the portal interrupt for FILTER mode (via + * XX_ConfigPortalNapi) — the FILTER handler inhibits the + * portal and enqueues the taskqueue task + * 3. Initialize the NCSW portal (Config + Stash + Init) + * 4. Subscribe to the pool channel so QMan distributes RX + * frames across all portals (round-robin) + * + * On ARM64, pmap_mapdev creates global kernel VA, so no + * sched_bind/migration is needed. + */ +t_Handle +qman_portal_init_cpu(struct qman_softc *qsc, int cpu) +{ + struct dpaa_portals_softc *sc; + t_QmPortalParam qpp; + t_QmPortalStashParam stash; + t_Handle portal; + cpuset_t cpumask; + + if (qp_sc == NULL) + return (NULL); + sc = qp_sc; + + /* Skip if already initialized */ + if (sc->sc_dp[cpu].dp_ph != NULL) + return (sc->sc_dp[cpu].dp_ph); + + /* Map CE/CI registers (global kernel VA, accessible from any CPU) */ + dpaa_portal_map_registers_cpu(sc, cpu); + + if (sc->sc_dp[cpu].dp_ce_va == 0 || sc->sc_dp[cpu].dp_ci_va == 0) + return (NULL); + + if (sc->sc_dp[cpu].dp_ires == NULL) { + printf("qman: portal %d has no IRQ resource\n", cpu); + return (NULL); + } + + /* Init deferred RX mbuf queue (unlimited — DQRR ring is the natural cap) */ + mbufq_init(&qman_napi[cpu].rxq, 0); + + /* + * Create per-CPU taskqueue for NAPI-style polling. + * The task arg is set to the portal handle after Init. + */ + CPU_ZERO(&cpumask); + CPU_SET(cpu, &cpumask); + NET_TASK_INIT(&qman_napi[cpu].task, 0, qman_portal_poll_task, NULL); + qman_napi[cpu].tq = taskqueue_create_fast("qman_poll", M_WAITOK, + taskqueue_thread_enqueue, &qman_napi[cpu].tq); + taskqueue_start_threads_cpuset(&qman_napi[cpu].tq, 1, PI_NET, + &cpumask, "qman%d", cpu); + + /* + * Configure NAPI mode before QM_PORTAL_Init — Init calls + * XX_SetIntr internally, which checks for NAPI mode and + * registers a FILTER handler instead of an ithread handler. + */ + XX_ConfigPortalNapi((uintptr_t)sc->sc_dp[cpu].dp_ires, + qman_napi[cpu].tq, &qman_napi[cpu].task); + + /* Configure portal */ + memset(&qpp, 0, sizeof(qpp)); + qpp.ceBaseAddress = sc->sc_dp[cpu].dp_ce_va; + qpp.ciBaseAddress = sc->sc_dp[cpu].dp_ci_va; + qpp.h_Qm = qsc->sc_qh; + qpp.swPortalId = cpu; + qpp.irq = (uintptr_t)sc->sc_dp[cpu].dp_ires; + qpp.fdLiodnOffset = 0; + qpp.f_DfltFrame = qman_received_frame_callback; + qpp.f_RejectedFrame = qman_rejected_frame_callback; + qpp.h_App = qsc; + + portal = QM_PORTAL_Config(&qpp); + if (portal == NULL) + return (NULL); + + /* + * Configure stash destination (SDEST) for this CPU. + * Linux: qman_set_sdest(channel, cpu_idx / 2) for QMan v3+. + * Each pair of cores shares one Stash Request Queue (SRQ) in + * the CoreNet fabric. LS1046A: SRQ0=CPU0-1, SRQ1=CPU2-3. + */ + memset(&stash, 0, sizeof(stash)); + stash.stashDestQueue = cpu / 2; + QM_PORTAL_ConfigStash(portal, &stash); + + if (QM_PORTAL_Init(portal) != E_OK) { + QM_PORTAL_Free(portal); + return (NULL); + } + + /* + * Set portal handle for the FILTER handler and task. + * The FILTER handler uses this to call QM_PORTAL_Inhibit. + * The task function receives it as the arg parameter. + */ + XX_SetPortalNapiHandle((uintptr_t)sc->sc_dp[cpu].dp_ires, portal); + qman_napi[cpu].task.ta_context = portal; + + /* + * Subscribe to pool channel — all portals get RX frames. + * QMan hardware distributes frames round-robin across all + * portals subscribed to the pool channel, matching Linux's + * dpaa_eth_add_channel() which calls qman_p_static_dequeue_add + * for every CPU's portal. + * + * RX FQs are on per-CPU dedicated channels (RSS), so pool + * channel round-robin no longer causes TCP reordering. Pool + * channel is still needed for TX confirm, CDX dist FQs, and + * CAAM QI response FQs. + */ + QM_PORTAL_AddPoolChannel(portal, QMAN_COMMON_POOL_CHANNEL); + + sc->sc_dp[cpu].dp_ph = portal; + return (portal); +} + +void +qman_portal_dqrr_diag(void) +{ + struct dpaa_portals_softc *sc; + t_Handle portal; + uint8_t sw_pi, sw_ci, sw_fill, hw_pi, hw_ci; + uint32_t isr, ier, iir; + int cpu; + + if (qp_sc == NULL) + return; + sc = qp_sc; + + for (cpu = 0; cpu < mp_ncpus; cpu++) { + portal = sc->sc_dp[cpu].dp_ph; + if (portal == NULL) + continue; + QM_PORTAL_DqrrDiag(portal, &sw_pi, &sw_ci, &sw_fill, + &hw_pi, &hw_ci); + QM_PORTAL_IsrDiag(portal, &isr, &ier, &iir); + printf("qman: portal%d: dqrr sw_pi=%u sw_ci=%u fill=%u " + "hw_pi=%u hw_ci=%u isr=0x%05x ier=0x%05x iir=%u\n", + cpu, sw_pi, sw_ci, sw_fill, hw_pi, hw_ci, + isr, ier, iir); + } +} +#endif diff --git a/sys/dev/gpio/gpiobus.c b/sys/dev/gpio/gpiobus.c index a8dd8a753676..ddb192ac3d51 100644 --- a/sys/dev/gpio/gpiobus.c +++ b/sys/dev/gpio/gpiobus.c @@ -40,6 +40,10 @@ #include #include +#ifdef FDT +#include +#include +#endif #include "gpiobus_if.h" @@ -293,6 +297,53 @@ gpiobus_print_pins(struct gpiobus_ivar *devi, struct sbuf *sb) sbuf_printf(sb, "%d", range_start); } +#ifdef FDT +/* + * Process a gpio-hog child node of a GPIO controller. + * Sets the pin direction and value as specified in the DT. + * The "gpios" property is controller-relative: . + * "output-high"/"output-low" are logical levels; GPIO_ACTIVE_LOW + * inverts the physical pin state. + */ +static void +gpiobus_process_hog(device_t dev, phandle_t child) +{ + char name[64]; + uint32_t gpiocells[2]; + uint32_t pin; + int active_low, val; + + if (OF_getencprop(child, "gpios", gpiocells, sizeof(gpiocells)) != + sizeof(gpiocells)) + return; + + pin = gpiocells[0]; + active_low = (gpiocells[1] & 1); /* bit 0 = GPIO_ACTIVE_LOW */ + + name[0] = '\0'; + OF_getprop(child, "line-name", name, sizeof(name)); + name[sizeof(name) - 1] = '\0'; + + if (OF_hasprop(child, "output-high")) { + val = active_low ? 0 : 1; + GPIO_PIN_SETFLAGS(dev, pin, GPIO_PIN_OUTPUT); + GPIO_PIN_SET(dev, pin, val); + device_printf(dev, "gpio-hog: pin %d (%s) " + "output-high (phys=%d)\n", pin, name, val); + } else if (OF_hasprop(child, "output-low")) { + val = active_low ? 1 : 0; + GPIO_PIN_SETFLAGS(dev, pin, GPIO_PIN_OUTPUT); + GPIO_PIN_SET(dev, pin, val); + device_printf(dev, "gpio-hog: pin %d (%s) " + "output-low (phys=%d)\n", pin, name, val); + } else if (OF_hasprop(child, "input")) { + GPIO_PIN_SETFLAGS(dev, pin, GPIO_PIN_INPUT); + device_printf(dev, "gpio-hog: pin %d (%s) input\n", + pin, name); + } +} +#endif + device_t gpiobus_attach_bus(device_t dev) { @@ -307,6 +358,23 @@ gpiobus_attach_bus(device_t dev) } #ifdef FDT ofw_gpiobus_register_provider(dev); + /* + * Process gpio-hog children before bus_generic_attach() so + * that muxes, power enables, and other board signals are + * configured before downstream devices (PCIe, UART) probe. + */ + { + phandle_t node, child; + + node = ofw_bus_get_node(dev); + if (node > 0) { + for (child = OF_child(node); child != 0; + child = OF_peer(child)) { + if (OF_hasprop(child, "gpio-hog")) + gpiobus_process_hog(dev, child); + } + } + } #endif bus_generic_attach(dev); diff --git a/sys/dev/gpio/gpiopower.c b/sys/dev/gpio/gpiopower.c index 2f09f79850d5..f8bd306ec41a 100644 --- a/sys/dev/gpio/gpiopower.c +++ b/sys/dev/gpio/gpiopower.c @@ -98,7 +98,7 @@ gpiopower_attach(device_t dev) sc->sc_timeout = fdt32_to_cpu(prop) * 1000; EVENTHANDLER_REGISTER(shutdown_final, gpiopower_assert, dev, - SHUTDOWN_PRI_LAST); + SHUTDOWN_PRI_LAST - 1); return (0); } diff --git a/sys/dev/gpio/ofw_gpiobus.c b/sys/dev/gpio/ofw_gpiobus.c index 7915562f226e..71f7066e70f7 100644 --- a/sys/dev/gpio/ofw_gpiobus.c +++ b/sys/dev/gpio/ofw_gpiobus.c @@ -430,6 +430,7 @@ ofw_gpiobus_attach(device_t dev) bus_enumerate_hinted_children(dev); /* * Attach the children represented in the device tree. + * gpio-hog nodes are already processed by gpiobus_attach_bus(). */ for (child = OF_child(ofw_bus_get_node(dev)); child != 0; child = OF_peer(child)) { diff --git a/sys/dev/gpio/qoriq_gpio.c b/sys/dev/gpio/qoriq_gpio.c index 5dabe03b775a..c67e704effd2 100644 --- a/sys/dev/gpio/qoriq_gpio.c +++ b/sys/dev/gpio/qoriq_gpio.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,28 @@ #include "gpio_if.h" +/* + * QorIQ GPIO registers are big-endian. bus_read_4/bus_write_4 return/accept + * data in CPU-native byte order. On PowerPC (BE) this is a no-op, but on + * ARM64 (LE) the bytes get swapped, which shifts bit positions within the + * 32-bit word and breaks the pin ↔ bit mapping. Wrap all register access + * with be32toh/htobe32 so pin numbering matches the hardware datasheet on + * both architectures. + */ +static inline uint32_t +gpio_read(struct qoriq_gpio_softc *sc, bus_size_t off) +{ + + return (be32toh(bus_read_4(sc->sc_mem, off))); +} + +static inline void +gpio_write(struct qoriq_gpio_softc *sc, bus_size_t off, uint32_t val) +{ + + bus_write_4(sc->sc_mem, off, htobe32(val)); +} + static device_t qoriq_gpio_get_bus(device_t dev) { @@ -112,20 +135,20 @@ qoriq_gpio_pin_configure(device_t dev, uint32_t pin, uint32_t flags) } if (flags & GPIO_PIN_INPUT) { - reg = bus_read_4(sc->sc_mem, GPIO_GPDIR); + reg = gpio_read(sc, GPIO_GPDIR); reg &= ~(1 << (31 - pin)); - bus_write_4(sc->sc_mem, GPIO_GPDIR, reg); + gpio_write(sc, GPIO_GPDIR, reg); } else if (flags & GPIO_PIN_OUTPUT) { - reg = bus_read_4(sc->sc_mem, GPIO_GPDIR); + reg = gpio_read(sc, GPIO_GPDIR); reg |= (1 << (31 - pin)); - bus_write_4(sc->sc_mem, GPIO_GPDIR, reg); - reg = bus_read_4(sc->sc_mem, GPIO_GPODR); + gpio_write(sc, GPIO_GPDIR, reg); + reg = gpio_read(sc, GPIO_GPODR); if (flags & GPIO_PIN_OPENDRAIN) reg |= (1 << (31 - pin)); else reg &= ~(1 << (31 - pin)); - bus_write_4(sc->sc_mem, GPIO_GPODR, reg); + gpio_write(sc, GPIO_GPODR, reg); } sc->sc_pins[pin].gp_flags = flags; @@ -183,10 +206,10 @@ qoriq_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) GPIO_LOCK(sc); pinbit = 31 - pin; - outvals = bus_read_4(sc->sc_mem, GPIO_GPDAT); + outvals = gpio_read(sc, GPIO_GPDAT); outvals &= ~(1 << pinbit); outvals |= (value << pinbit); - bus_write_4(sc->sc_mem, GPIO_GPDAT, outvals); + gpio_write(sc, GPIO_GPDAT, outvals); GPIO_UNLOCK(sc); @@ -202,7 +225,7 @@ qoriq_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *value) if (!VALID_PIN(pin)) return (EINVAL); - *value = (bus_read_4(sc->sc_mem, GPIO_GPDAT) >> (31 - pin)) & 1; + *value = (gpio_read(sc, GPIO_GPDAT) >> (31 - pin)) & 1; return (0); } @@ -219,9 +242,9 @@ qoriq_gpio_pin_toggle(device_t dev, uint32_t pin) GPIO_LOCK(sc); - val = bus_read_4(sc->sc_mem, GPIO_GPDAT); + val = gpio_read(sc, GPIO_GPDAT); val ^= (1 << (31 - pin)); - bus_write_4(sc->sc_mem, GPIO_GPDAT, val); + gpio_write(sc, GPIO_GPDAT, val); GPIO_UNLOCK(sc); @@ -260,8 +283,8 @@ qoriq_gpio_pin_access_32(device_t dev, uint32_t first_pin, uint32_t clear_pins, return (EINVAL); GPIO_LOCK(sc); - hwstate = bus_read_4(sc->sc_mem, GPIO_GPDAT); - bus_write_4(sc->sc_mem, GPIO_GPDAT, + hwstate = gpio_read(sc, GPIO_GPDAT); + gpio_write(sc, GPIO_GPDAT, (hwstate & ~clear_pins) ^ change_pins); GPIO_UNLOCK(sc); @@ -310,11 +333,11 @@ qoriq_gpio_pin_config_32(device_t dev, uint32_t first_pin, uint32_t num_pins, GPIO_LOCK(sc); - reg = (bus_read_4(sc->sc_mem, GPIO_GPDIR) & ~mask) | dir; - bus_write_4(sc->sc_mem, GPIO_GPDIR, reg); + reg = (gpio_read(sc, GPIO_GPDIR) & ~mask) | dir; + gpio_write(sc, GPIO_GPDIR, reg); - reg = (bus_read_4(sc->sc_mem, GPIO_GPODR) & ~mask) | odr; - bus_write_4(sc->sc_mem, GPIO_GPODR, reg); + reg = (gpio_read(sc, GPIO_GPODR) & ~mask) | odr; + gpio_write(sc, GPIO_GPODR, reg); for (i = 0; i < num_pins; i++) sc->sc_pins[i].gp_flags = newflags[i]; @@ -381,7 +404,7 @@ qoriq_gpio_attach(device_t dev) * devices ignore writes and read 0's in undefined portions of the map. */ if (ofw_bus_is_compatible(dev, "fsl,qoriq-gpio")) - bus_write_4(sc->sc_mem, GPIO_GPIBE, 0xffffffff); + gpio_write(sc, GPIO_GPIBE, 0xffffffff); OF_device_register_xref(OF_xref_from_node(ofw_bus_get_node(dev)), dev); diff --git a/sys/dev/iicbus/mux/iicmux.c b/sys/dev/iicbus/mux/iicmux.c index 5eafc8a5a21b..36a405f6f32d 100644 --- a/sys/dev/iicbus/mux/iicmux.c +++ b/sys/dev/iicbus/mux/iicmux.c @@ -279,7 +279,8 @@ iicmux_attach_children(struct iicmux_softc *sc) idx, sc->numbuses); continue; } - sc->childdevs[idx] = device_add_child(sc->dev, "iicbus", -1); + sc->childdevs[idx] = device_add_child(sc->dev, + "iicbus", -1); sc->childnodes[idx] = child; if (sc->maxbus < (int)idx) sc->maxbus = idx; diff --git a/sys/dev/iicbus/mux/pca954x.c b/sys/dev/iicbus/mux/pca954x.c index befa0db39172..8a8ae3cf05de 100644 --- a/sys/dev/iicbus/mux/pca954x.c +++ b/sys/dev/iicbus/mux/pca954x.c @@ -94,6 +94,7 @@ static struct pca954x_descr pca9548_descr = { #ifdef FDT static struct ofw_compat_data compat_data[] = { { "nxp,pca9540", (uintptr_t)&pca9540_descr }, + { "nxp,pca9545", (uintptr_t)&pca9546_descr }, { "nxp,pca9546", (uintptr_t)&pca9546_descr }, { "nxp,pca9547", (uintptr_t)&pca9547_descr }, { "nxp,pca9548", (uintptr_t)&pca9548_descr }, diff --git a/sys/dev/iicbus/ofw_iicbus.c b/sys/dev/iicbus/ofw_iicbus.c index fdf7e0e188f4..f2c0d41b294c 100644 --- a/sys/dev/iicbus/ofw_iicbus.c +++ b/sys/dev/iicbus/ofw_iicbus.c @@ -125,7 +125,7 @@ ofw_iicbus_attach(device_t dev) freq = 0; OF_getencprop(node, "clock-frequency", &freq, sizeof(freq)); iicbus_init_frequency(dev, freq); - + iicbus_reset(dev, IIC_FASTEST, 0, NULL); bus_generic_probe(dev); diff --git a/sys/dev/iicbus/sensor/ina2xx.c b/sys/dev/iicbus/sensor/ina2xx.c new file mode 100644 index 000000000000..1554ba75236f --- /dev/null +++ b/sys/dev/iicbus/sensor/ina2xx.c @@ -0,0 +1,399 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Mono Technologies Inc. + * + * Texas Instruments INA219/INA226/INA234 I2C power monitor driver. + * Reads bus voltage, shunt voltage, current, and power from + * INA2xx-family digital power monitors over I2C. + * + * Based on Linux hwmon/ina2xx.c by Lothar Felten. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* INA2xx register addresses */ +#define INA2XX_CONFIG 0x00 +#define INA2XX_SHUNT_VOLTAGE 0x01 +#define INA2XX_BUS_VOLTAGE 0x02 +#define INA2XX_POWER 0x03 +#define INA2XX_CURRENT 0x04 +#define INA2XX_CALIBRATION 0x05 + +/* Default shunt resistance: 10 mOhm = 10000 uOhm */ +#define INA2XX_RSHUNT_DEFAULT 10000 + +/* Chip type enumeration */ +enum ina2xx_chip { INA219, INA226, INA234 }; + +/* Per-chip-type configuration constants */ +struct ina2xx_config { + uint16_t config_default; + uint16_t calibration_value; + int shunt_div; /* shunt voltage divider (LSB factor) */ + int bus_voltage_shift; + int bus_voltage_lsb; /* uV per LSB */ + int power_lsb_factor; +}; + +static const struct ina2xx_config ina2xx_configs[] = { + [INA219] = { + .config_default = 0x399F, /* PGA /8 */ + .calibration_value = 4096, + .shunt_div = 100, + .bus_voltage_shift = 3, + .bus_voltage_lsb = 4000, /* 4 mV */ + .power_lsb_factor = 20, + }, + [INA226] = { + .config_default = 0x4527, /* avg=16 */ + .calibration_value = 2048, + .shunt_div = 400, + .bus_voltage_shift = 0, + .bus_voltage_lsb = 1250, /* 1.25 mV */ + .power_lsb_factor = 25, + }, + [INA234] = { + .config_default = 0x4527, /* same as INA226 */ + .calibration_value = 2048, + .shunt_div = 400, + .bus_voltage_shift = 4, + .bus_voltage_lsb = 25600, /* 25.6 mV */ + .power_lsb_factor = 32, + }, +}; + +struct ina2xx_softc { + device_t dev; + enum ina2xx_chip chip; + const struct ina2xx_config *config; + uint32_t rshunt; /* shunt resistance, uOhm */ + long current_lsb_uA; + long power_lsb_uW; + struct sysctl_ctx_list sysctl_ctx; + char label[64]; +}; + +/* ================================================================ + * I2C register helpers — 16-bit big-endian + * ================================================================ */ + +static int +ina2xx_read_reg(device_t dev, uint8_t reg, uint16_t *val) +{ + uint8_t buf[2]; + int error; + + error = iicdev_readfrom(dev, reg, buf, 2, IIC_WAIT); + if (error != 0) + return (error); + *val = (buf[0] << 8) | buf[1]; + return (0); +} + +static int +ina2xx_write_reg(device_t dev, uint8_t reg, uint16_t val) +{ + uint8_t buf[2]; + + buf[0] = (val >> 8) & 0xFF; + buf[1] = val & 0xFF; + return (iicdev_writeto(dev, reg, buf, 2, IIC_WAIT)); +} + +/* ================================================================ + * Measurement value conversion (matches Linux ina2xx_get_value) + * ================================================================ */ + +static int +ina2xx_read_bus_voltage(struct ina2xx_softc *sc, int *millivolts) +{ + uint16_t raw; + int error; + + error = ina2xx_read_reg(sc->dev, INA2XX_BUS_VOLTAGE, &raw); + if (error != 0) + return (error); + *millivolts = ((raw >> sc->config->bus_voltage_shift) * + sc->config->bus_voltage_lsb) / 1000; + return (0); +} + +static int +ina2xx_read_shunt_voltage(struct ina2xx_softc *sc, int *microvolts) +{ + uint16_t raw; + int error; + + error = ina2xx_read_reg(sc->dev, INA2XX_SHUNT_VOLTAGE, &raw); + if (error != 0) + return (error); + /* Shunt voltage register is signed */ + *microvolts = (int16_t)raw * 1000 / sc->config->shunt_div; + return (0); +} + +static int +ina2xx_read_current(struct ina2xx_softc *sc, int *milliamps) +{ + uint16_t raw; + int error; + + error = ina2xx_read_reg(sc->dev, INA2XX_CURRENT, &raw); + if (error != 0) + return (error); + /* Current register is signed */ + *milliamps = (int16_t)raw * sc->current_lsb_uA / 1000; + return (0); +} + +static int +ina2xx_read_power(struct ina2xx_softc *sc, int *milliwatts) +{ + uint16_t raw; + int error; + + error = ina2xx_read_reg(sc->dev, INA2XX_POWER, &raw); + if (error != 0) + return (error); + /* Power register is unsigned */ + *milliwatts = (int)((uint32_t)raw * sc->power_lsb_uW / 1000); + return (0); +} + +/* ================================================================ + * Sysctl handlers + * ================================================================ */ + +static int +ina2xx_sysctl_bus_voltage(SYSCTL_HANDLER_ARGS) +{ + struct ina2xx_softc *sc = arg1; + int val, error; + + if (req->newptr != NULL) + return (EINVAL); + error = ina2xx_read_bus_voltage(sc, &val); + if (error != 0) + return (error); + return (sysctl_handle_int(oidp, &val, 0, req)); +} + +static int +ina2xx_sysctl_shunt_voltage(SYSCTL_HANDLER_ARGS) +{ + struct ina2xx_softc *sc = arg1; + int val, error; + + if (req->newptr != NULL) + return (EINVAL); + error = ina2xx_read_shunt_voltage(sc, &val); + if (error != 0) + return (error); + return (sysctl_handle_int(oidp, &val, 0, req)); +} + +static int +ina2xx_sysctl_current(SYSCTL_HANDLER_ARGS) +{ + struct ina2xx_softc *sc = arg1; + int val, error; + + if (req->newptr != NULL) + return (EINVAL); + error = ina2xx_read_current(sc, &val); + if (error != 0) + return (error); + return (sysctl_handle_int(oidp, &val, 0, req)); +} + +static int +ina2xx_sysctl_power(SYSCTL_HANDLER_ARGS) +{ + struct ina2xx_softc *sc = arg1; + int val, error; + + if (req->newptr != NULL) + return (EINVAL); + error = ina2xx_read_power(sc, &val); + if (error != 0) + return (error); + return (sysctl_handle_int(oidp, &val, 0, req)); +} + +/* ================================================================ + * Probe / Attach / Detach + * ================================================================ */ + +static struct ofw_compat_data compat_data[] = { + { "ti,ina219", (uintptr_t)INA219 }, + { "ti,ina220", (uintptr_t)INA219 }, + { "ti,ina226", (uintptr_t)INA226 }, + { "ti,ina230", (uintptr_t)INA226 }, + { "ti,ina231", (uintptr_t)INA226 }, + { "ti,ina234", (uintptr_t)INA234 }, + { NULL, 0 }, +}; + +static int +ina2xx_probe(device_t dev) +{ + const struct ofw_compat_data *cd; + phandle_t node; + char desc[80]; + char label[64]; + + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + cd = ofw_bus_search_compatible(dev, compat_data); + if (cd->ocd_data == 0) + return (ENXIO); + + node = ofw_bus_get_node(dev); + if (OF_getprop(node, "label", label, sizeof(label)) > 0) { + label[sizeof(label) - 1] = '\0'; + snprintf(desc, sizeof(desc), "INA2xx Power Monitor (%s)", + label); + } + else + snprintf(desc, sizeof(desc), "INA2xx Power Monitor"); + device_set_desc_copy(dev, desc); + + return (BUS_PROBE_DEFAULT); +} + +static int +ina2xx_attach(device_t dev) +{ + struct ina2xx_softc *sc; + const struct ofw_compat_data *cd; + phandle_t node; + struct sysctl_oid *oid; + uint32_t shunt; + long dividend; + int error; + + sc = device_get_softc(dev); + sc->dev = dev; + + cd = ofw_bus_search_compatible(dev, compat_data); + sc->chip = (enum ina2xx_chip)cd->ocd_data; + sc->config = &ina2xx_configs[sc->chip]; + + node = ofw_bus_get_node(dev); + + /* Read shunt resistance from DT (micro-ohms), default 10 mOhm */ + if (OF_getencprop(node, "shunt-resistor", &shunt, sizeof(shunt)) <= 0) + shunt = INA2XX_RSHUNT_DEFAULT; + sc->rshunt = shunt; + if (sc->rshunt == 0) + sc->rshunt = INA2XX_RSHUNT_DEFAULT; + + /* Read label from DT */ + sc->label[0] = '\0'; + OF_getprop(node, "label", sc->label, sizeof(sc->label)); + sc->label[sizeof(sc->label) - 1] = '\0'; + + /* Compute current and power LSBs */ + dividend = 1000000000L / sc->config->shunt_div; + sc->current_lsb_uA = dividend / (long)sc->rshunt; + sc->power_lsb_uW = sc->config->power_lsb_factor * sc->current_lsb_uA; + + /* Program configuration register */ + error = ina2xx_write_reg(dev, INA2XX_CONFIG, + sc->config->config_default); + if (error != 0) { + device_printf(dev, "failed to write config register\n"); + return (ENXIO); + } + + /* Program calibration register */ + error = ina2xx_write_reg(dev, INA2XX_CALIBRATION, + sc->config->calibration_value); + if (error != 0) { + device_printf(dev, "failed to write calibration register\n"); + return (ENXIO); + } + + /* Create sysctl tree */ + sysctl_ctx_init(&sc->sysctl_ctx); + oid = device_get_sysctl_tree(dev); + + if (sc->label[0] != '\0') { + SYSCTL_ADD_STRING(&sc->sysctl_ctx, SYSCTL_CHILDREN(oid), + OID_AUTO, "label", CTLFLAG_RD, sc->label, 0, + "Sensor label"); + } + + SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(oid), + OID_AUTO, "bus_voltage", + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, + sc, 0, ina2xx_sysctl_bus_voltage, "I", + "Bus voltage (mV)"); + + SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(oid), + OID_AUTO, "shunt_voltage", + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, + sc, 0, ina2xx_sysctl_shunt_voltage, "I", + "Shunt voltage (uV)"); + + SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(oid), + OID_AUTO, "current", + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, + sc, 0, ina2xx_sysctl_current, "I", + "Current (mA)"); + + SYSCTL_ADD_PROC(&sc->sysctl_ctx, SYSCTL_CHILDREN(oid), + OID_AUTO, "power", + CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, + sc, 0, ina2xx_sysctl_power, "I", + "Power (mW)"); + + if (sc->label[0] != '\0') + device_printf(dev, "%s: shunt=%u uOhm, " + "current_lsb=%ld uA, power_lsb=%ld uW\n", + sc->label, sc->rshunt, + sc->current_lsb_uA, sc->power_lsb_uW); + + return (0); +} + +static int +ina2xx_detach(device_t dev) +{ + struct ina2xx_softc *sc; + + sc = device_get_softc(dev); + sysctl_ctx_free(&sc->sysctl_ctx); + return (0); +} + +static device_method_t ina2xx_methods[] = { + DEVMETHOD(device_probe, ina2xx_probe), + DEVMETHOD(device_attach, ina2xx_attach), + DEVMETHOD(device_detach, ina2xx_detach), + DEVMETHOD_END +}; + +static driver_t ina2xx_driver = { + "ina2xx", + ina2xx_methods, + sizeof(struct ina2xx_softc), +}; + +DRIVER_MODULE(ina2xx, iicbus, ina2xx_driver, NULL, NULL); +MODULE_VERSION(ina2xx, 1); +MODULE_DEPEND(ina2xx, iicbus, IICBUS_MODVER, IICBUS_MODVER, IICBUS_MODVER); +IICBUS_FDT_PNP_INFO(compat_data); diff --git a/sys/dev/mii/gpyphy.c b/sys/dev/mii/gpyphy.c new file mode 100644 index 000000000000..4df46c0975f9 --- /dev/null +++ b/sys/dev/mii/gpyphy.c @@ -0,0 +1,340 @@ +/*- + * Copyright (c) 2026 Mono Technologies Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * MaxLinear GPY115/GPY211/GPY212/GPY215/GPY241/GPY245 PHY driver. + * + * These PHYs require explicit SGMII auto-negotiation enable via a + * vendor-specific MMD register (VSPEC1_SGMII_CTRL). Without this, + * the MAC-side PCS never synchronizes and the link stays down. + * + * Reference: Linux drivers/net/phy/mxl-gpy.c + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include "miidevs.h" + +#include "miibus_if.h" + +/* Vendor-specific Clause 22 registers */ +#define GPY_MIISTAT 0x18 /* MII state */ +#define GPY_IMASK 0x19 /* Interrupt mask */ +#define GPY_ISTAT 0x1A /* Interrupt status */ +#define GPY_FWV 0x1E /* Firmware version */ + +/* GPY_MIISTAT bits */ +#define GPY_MIISTAT_LS (1 << 10) /* Link status */ +#define GPY_MIISTAT_DPX (1 << 3) /* Duplex: 1=full */ +#define GPY_MIISTAT_SPD_MASK 0x0007 +#define GPY_MIISTAT_SPD_10 0 +#define GPY_MIISTAT_SPD_100 1 +#define GPY_MIISTAT_SPD_1000 2 +#define GPY_MIISTAT_SPD_2500 4 + +/* GPY_FWV bits */ +#define GPY_FWV_REL_MASK (1 << 15) +#define GPY_FWV_MAJOR_MASK 0x0F00 +#define GPY_FWV_MAJOR_SHIFT 8 +#define GPY_FWV_MINOR_MASK 0x00FF + +/* MMD device address for vendor-specific registers */ +#define MDIO_MMD_VEND1 30 + +/* VSPEC1 (MMD30) registers */ +#define VSPEC1_SGMII_CTRL 0x08 +#define VSPEC1_SGMII_CTRL_ANEN (1 << 12) /* SGMII AN enable */ +#define VSPEC1_SGMII_CTRL_ANRS (1 << 9) /* SGMII AN restart */ + +struct gpyphy_softc { + struct mii_softc mii_sc; + uint8_t fw_major; + uint8_t fw_minor; + int prev_link; +}; + +static int gpyphy_probe(device_t); +static int gpyphy_attach(device_t); +static int gpyphy_service(struct mii_softc *, struct mii_data *, int); +static void gpyphy_status(struct mii_softc *); +static void gpyphy_reset(struct mii_softc *); + +static device_method_t gpyphy_methods[] = { + DEVMETHOD(device_probe, gpyphy_probe), + DEVMETHOD(device_attach, gpyphy_attach), + DEVMETHOD(device_detach, mii_phy_detach), + DEVMETHOD(device_shutdown, bus_generic_shutdown), + DEVMETHOD_END +}; + +static driver_t gpyphy_driver = { + "gpyphy", + gpyphy_methods, + sizeof(struct gpyphy_softc) +}; + +DRIVER_MODULE(gpyphy, miibus, gpyphy_driver, 0, 0); + +static const struct mii_phydesc gpyphys[] = { + MII_PHY_DESC(MAXLINEAR, GPY2xx), + MII_PHY_DESC(MAXLINEAR, GPY115), + MII_PHY_DESC(MAXLINEAR, GPY115C), + MII_PHY_DESC(MAXLINEAR, GPY211), + MII_PHY_DESC(MAXLINEAR, GPY211C), + MII_PHY_DESC(MAXLINEAR, GPY212C), + MII_PHY_DESC(MAXLINEAR, GPY215C), + MII_PHY_DESC(MAXLINEAR, GPY241), + MII_PHY_DESC(MAXLINEAR, GPY241BM), + MII_PHY_DESC(MAXLINEAR, GPY245), + MII_PHY_END +}; + +static const struct mii_phy_funcs gpyphy_funcs = { + gpyphy_service, + gpyphy_status, + gpyphy_reset +}; + +/* + * MMD indirect access via Clause 22 registers 13/14 (MII_MMDACR/MII_MMDAADR). + * Same pattern used by micphy.c (ksz9031_read/write) and rgephy.c. + */ +static int +gpyphy_mmd_read(struct mii_softc *sc, int devaddr, int reg) +{ + + /* Set MMD device address */ + PHY_WRITE(sc, MII_MMDACR, devaddr & MMDACR_DADDRMASK); + /* Set MMD register address */ + PHY_WRITE(sc, MII_MMDAADR, reg); + /* Select data, no post-increment */ + PHY_WRITE(sc, MII_MMDACR, + MMDACR_FN_DATANPI | (devaddr & MMDACR_DADDRMASK)); + /* Read data */ + return (PHY_READ(sc, MII_MMDAADR)); +} + +static void +gpyphy_mmd_write(struct mii_softc *sc, int devaddr, int reg, int val) +{ + + /* Set MMD device address */ + PHY_WRITE(sc, MII_MMDACR, devaddr & MMDACR_DADDRMASK); + /* Set MMD register address */ + PHY_WRITE(sc, MII_MMDAADR, reg); + /* Select data, no post-increment */ + PHY_WRITE(sc, MII_MMDACR, + MMDACR_FN_DATANPI | (devaddr & MMDACR_DADDRMASK)); + /* Write data */ + PHY_WRITE(sc, MII_MMDAADR, val); +} + +/* + * Enable and restart SGMII auto-negotiation. + * + * The GPY PHY uses SGMII to communicate with the MAC's PCS. + * Without SGMII AN enabled, the PCS never synchronizes and the + * MAC reports no carrier even when the copper link is up. + */ +static void +gpyphy_sgmii_aneg(struct mii_softc *sc) +{ + int val; + + val = gpyphy_mmd_read(sc, MDIO_MMD_VEND1, VSPEC1_SGMII_CTRL); + val |= VSPEC1_SGMII_CTRL_ANEN | VSPEC1_SGMII_CTRL_ANRS; + gpyphy_mmd_write(sc, MDIO_MMD_VEND1, VSPEC1_SGMII_CTRL, val); +} + +static int +gpyphy_probe(device_t dev) +{ + + return (mii_phy_dev_probe(dev, gpyphys, BUS_PROBE_DEFAULT)); +} + +static int +gpyphy_attach(device_t dev) +{ + struct gpyphy_softc *gsc; + struct mii_softc *sc; + int fwv; + + gsc = device_get_softc(dev); + sc = &gsc->mii_sc; + gsc->prev_link = 0; + + /* Must initialize mii_softc before any PHY register access */ + mii_phy_dev_attach(dev, MIIF_NOMANPAUSE, &gpyphy_funcs, 1); + + /* Read firmware version */ + fwv = PHY_READ(sc, GPY_FWV); + gsc->fw_major = (fwv & GPY_FWV_MAJOR_MASK) >> GPY_FWV_MAJOR_SHIFT; + gsc->fw_minor = fwv & GPY_FWV_MINOR_MASK; + + device_printf(dev, "MaxLinear GPY Firmware: %d.%d (0x%04x%s)\n", + gsc->fw_major, gsc->fw_minor, fwv, + (fwv & GPY_FWV_REL_MASK) ? "" : " test"); + + /* Enable SGMII auto-negotiation — critical for MAC-side link */ + gpyphy_sgmii_aneg(sc); + + mii_phy_setmedia(sc); + + return (0); +} + +static int +gpyphy_service(struct mii_softc *sc, struct mii_data *mii, int cmd) +{ + struct gpyphy_softc *gsc = (struct gpyphy_softc *)sc; + + switch (cmd) { + case MII_POLLSTAT: + break; + + case MII_MEDIACHG: + mii_phy_setmedia(sc); + /* Re-trigger SGMII AN after media change */ + gpyphy_sgmii_aneg(sc); + break; + + case MII_TICK: + if (mii_phy_tick(sc) == EJUSTRETURN) + return (0); + break; + } + + /* Update the media status */ + PHY_STATUS(sc); + + /* + * Re-trigger SGMII AN on link-up transitions. + * + * Some GPY firmware versions don't re-trigger SGMII AN when the + * copper link comes back at the same speed. Always re-trigger + * on down→up transitions to be safe — SGMII AN restart is + * harmless if already negotiated. + */ + if ((mii->mii_media_status & IFM_ACTIVE) && !gsc->prev_link) { + gpyphy_sgmii_aneg(sc); + } + gsc->prev_link = (mii->mii_media_status & IFM_ACTIVE) ? 1 : 0; + + /* Callback if something changed */ + mii_phy_update(sc, cmd); + return (0); +} + +static void +gpyphy_status(struct mii_softc *sc) +{ + struct mii_data *mii = sc->mii_pdata; + int bmsr, miistat; + + mii->mii_media_status = IFM_AVALID; + mii->mii_media_active = IFM_ETHER; + + /* Read BMSR twice — link status bit latches low */ + bmsr = PHY_READ(sc, MII_BMSR) | PHY_READ(sc, MII_BMSR); + + /* + * Read vendor-specific MIISTAT for link, speed, and duplex. + * PHY_MIISTAT reflects the actual resolved link state and is + * more reliable than decoding ANLPAR for GPY PHYs. + */ + miistat = PHY_READ(sc, GPY_MIISTAT); + + if (!(miistat & GPY_MIISTAT_LS)) { + /* No link — check BMSR too for consistency */ + if (bmsr & BMSR_LINK) { + /* + * BMSR shows link but MIISTAT doesn't — this can + * happen briefly during SGMII negotiation. Report + * no link; the next tick will re-check. + */ + } + mii->mii_media_active |= IFM_NONE; + return; + } + + mii->mii_media_status |= IFM_ACTIVE; + + /* Duplex */ + if (miistat & GPY_MIISTAT_DPX) + mii->mii_media_active |= IFM_FDX; + else + mii->mii_media_active |= IFM_HDX; + + /* Speed */ + switch (miistat & GPY_MIISTAT_SPD_MASK) { + case GPY_MIISTAT_SPD_1000: + mii->mii_media_active |= IFM_1000_T; + break; + case GPY_MIISTAT_SPD_100: + mii->mii_media_active |= IFM_100_TX; + break; + case GPY_MIISTAT_SPD_10: + mii->mii_media_active |= IFM_10_T; + break; + case GPY_MIISTAT_SPD_2500: + mii->mii_media_active |= IFM_2500_T; + break; + default: + mii->mii_media_active |= IFM_NONE; + break; + } + + if ((mii->mii_media_active & IFM_FDX) != 0) + mii->mii_media_active |= mii_phy_flowstatus(sc); +} + +static void +gpyphy_reset(struct mii_softc *sc) +{ + + /* + * GPY is a firmware-based PHY. BMCR software reset disrupts + * firmware operation (copper link never establishes, LEDs stay + * dark). Linux's GPY driver does not define .soft_reset either. + * + * Skip mii_phy_reset() and just ensure SGMII AN is enabled. + * mii_phy_setmedia() calls PHY_RESET before mii_phy_auto(), + * so this path is hit on every media change. + */ + gpyphy_sgmii_aneg(sc); +} diff --git a/sys/dev/mii/miidevs b/sys/dev/mii/miidevs index 78d1dddba59f..e28154eaef8a 100644 --- a/sys/dev/mii/miidevs +++ b/sys/dev/mii/miidevs @@ -61,6 +61,7 @@ oui INTEL 0x00aa00 Intel Corporation oui JMICRON 0x00d831 JMicron Technologies oui LEVEL1 0x00207b Level 1 oui MARVELL 0x005043 Marvell Semiconductor +oui MAXLINEAR 0x984fee MaxLinear Inc. oui MICREL 0x0010a1 Micrel oui MOTORCOMM 0x000000 Motorcomm oui MYSON 0x00c0b4 Myson Technology @@ -288,6 +289,18 @@ model MARVELL E1000_3 0x0003 Marvell 88E1000 Gigabit PHY model MARVELL E1000_5 0x0005 Marvell 88E1000 Gigabit PHY model MARVELL E1111 0x000c Marvell 88E1111 Gigabit PHY +/* MaxLinear GPY PHYs */ +model MAXLINEAR GPY2xx 0x0000 MaxLinear GPY2xx 10/100/1000 PHY +model MAXLINEAR GPY115 0x0030 MaxLinear GPY115 10/100/1000 PHY +model MAXLINEAR GPY115C 0x0031 MaxLinear GPY115C 10/100/1000 PHY +model MAXLINEAR GPY211 0x0020 MaxLinear GPY211 10/100/1000 PHY +model MAXLINEAR GPY211C 0x0021 MaxLinear GPY211C 10/100/1000 PHY +model MAXLINEAR GPY212C 0x0022 MaxLinear GPY212C 10/100/1000 PHY +model MAXLINEAR GPY215C 0x0032 MaxLinear GPY215C 10/100/1000 PHY +model MAXLINEAR GPY241 0x0024 MaxLinear GPY241 10/100/1000 PHY +model MAXLINEAR GPY241BM 0x0028 MaxLinear GPY241BM 10/100/1000 PHY +model MAXLINEAR GPY245 0x002c MaxLinear GPY245 10/100/1000 PHY + /* Micrel PHYs */ model MICREL KSZ8081 0x0016 Micrel KSZ8081 10/100 PHY model MICREL KSZ9021 0x0021 Micrel KSZ9021 10/100/1000 PHY diff --git a/sys/dev/pci/pci_dw.c b/sys/dev/pci/pci_dw.c index eae90e8f5f9e..e7fcc9a47eb4 100644 --- a/sys/dev/pci/pci_dw.c +++ b/sys/dev/pci/pci_dw.c @@ -366,64 +366,79 @@ pci_dw_setup_hw(struct pci_dw_softc *sc) return (rv); } - /* Adjust number of lanes */ - reg = DBI_RD4(sc, DW_PORT_LINK_CTRL); - reg &= ~PORT_LINK_CAPABLE(~0); - switch (sc->num_lanes) { - case 1: - reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_1); - break; - case 2: - reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_2); - break; - case 4: - reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_4); - break; - case 8: - reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_8); - break; - case 16: - reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_16); - break; - case 32: - reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_32); - break; - default: - device_printf(sc->dev, - "'num-lanes' property have invalid value: %d\n", - sc->num_lanes); - return (EINVAL); - } - DBI_WR4(sc, DW_PORT_LINK_CTRL, reg); + /* + * Adjust number of lanes and trigger link retraining only if + * "num-lanes" is explicitly specified in DT. When absent + * (num_lanes == 0), preserve the link configuration established + * by firmware/bootloader. This matches Linux behavior where + * dw_pcie_link_set_max_link_width() returns early if num_lanes + * is zero. + */ + if (sc->num_lanes > 0) { + reg = DBI_RD4(sc, DW_PORT_LINK_CTRL); + reg &= ~PORT_LINK_CAPABLE(~0); + switch (sc->num_lanes) { + case 1: + reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_1); + break; + case 2: + reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_2); + break; + case 4: + reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_4); + break; + case 8: + reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_8); + break; + case 16: + reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_16); + break; + case 32: + reg |= PORT_LINK_CAPABLE(PORT_LINK_CAPABLE_32); + break; + default: + device_printf(sc->dev, + "'num-lanes' property have invalid value: %d\n", + sc->num_lanes); + return (EINVAL); + } + DBI_WR4(sc, DW_PORT_LINK_CTRL, reg); + + /* And link width */ + reg = DBI_RD4(sc, DW_GEN2_CTRL); + reg &= ~GEN2_CTRL_NUM_OF_LANES(~0); + switch (sc->num_lanes) { + case 1: + reg |= GEN2_CTRL_NUM_OF_LANES( + GEN2_CTRL_NUM_OF_LANES_1); + break; + case 2: + reg |= GEN2_CTRL_NUM_OF_LANES( + GEN2_CTRL_NUM_OF_LANES_2); + break; + case 4: + reg |= GEN2_CTRL_NUM_OF_LANES( + GEN2_CTRL_NUM_OF_LANES_4); + break; + case 8: + reg |= GEN2_CTRL_NUM_OF_LANES( + GEN2_CTRL_NUM_OF_LANES_8); + break; + case 16: + reg |= GEN2_CTRL_NUM_OF_LANES( + GEN2_CTRL_NUM_OF_LANES_16); + break; + case 32: + reg |= GEN2_CTRL_NUM_OF_LANES( + GEN2_CTRL_NUM_OF_LANES_32); + break; + } + DBI_WR4(sc, DW_GEN2_CTRL, reg); - /* And link width */ - reg = DBI_RD4(sc, DW_GEN2_CTRL); - reg &= ~GEN2_CTRL_NUM_OF_LANES(~0); - switch (sc->num_lanes) { - case 1: - reg |= GEN2_CTRL_NUM_OF_LANES(GEN2_CTRL_NUM_OF_LANES_1); - break; - case 2: - reg |= GEN2_CTRL_NUM_OF_LANES(GEN2_CTRL_NUM_OF_LANES_2); - break; - case 4: - reg |= GEN2_CTRL_NUM_OF_LANES(GEN2_CTRL_NUM_OF_LANES_4); - break; - case 8: - reg |= GEN2_CTRL_NUM_OF_LANES(GEN2_CTRL_NUM_OF_LANES_8); - break; - case 16: - reg |= GEN2_CTRL_NUM_OF_LANES(GEN2_CTRL_NUM_OF_LANES_16); - break; - case 32: - reg |= GEN2_CTRL_NUM_OF_LANES(GEN2_CTRL_NUM_OF_LANES_32); - break; + reg = DBI_RD4(sc, DW_GEN2_CTRL); + reg |= DIRECT_SPEED_CHANGE; + DBI_WR4(sc, DW_GEN2_CTRL, reg); } - DBI_WR4(sc, DW_GEN2_CTRL, reg); - - reg = DBI_RD4(sc, DW_GEN2_CTRL); - reg |= DIRECT_SPEED_CHANGE; - DBI_WR4(sc, DW_GEN2_CTRL, reg); return (0); } @@ -725,8 +740,8 @@ pci_dw_init(device_t dev) rv = OF_getencprop(sc->node, "num-lanes", &sc->num_lanes, sizeof(sc->num_lanes)); if (rv != sizeof(sc->num_lanes)) - sc->num_lanes = 1; - if (sc->num_lanes != 1 && sc->num_lanes != 2 && + sc->num_lanes = 0; + if (sc->num_lanes != 0 && sc->num_lanes != 1 && sc->num_lanes != 2 && sc->num_lanes != 4 && sc->num_lanes != 8) { device_printf(dev, "invalid number of lanes: %d\n",sc->num_lanes); diff --git a/sys/dev/random/random_harvestq.c b/sys/dev/random/random_harvestq.c index 2a218970b850..f37f32e0f04d 100644 --- a/sys/dev/random/random_harvestq.c +++ b/sys/dev/random/random_harvestq.c @@ -387,6 +387,7 @@ static const char *random_source_descr[ENTROPYSOURCE] = { [RANDOM_PURE_VMGENID] = "PURE_VMGENID", [RANDOM_PURE_QUALCOMM] = "PURE_QUALCOMM", [RANDOM_PURE_ARMV8] = "PURE_ARMV8", + [RANDOM_PURE_CAAM] = "PURE_CAAM", /* "ENTROPYSOURCE" */ }; diff --git a/sys/dev/sff/sff_if.m b/sys/dev/sff/sff_if.m index 823e557992c2..0048114047c6 100644 --- a/sys/dev/sff/sff_if.m +++ b/sys/dev/sff/sff_if.m @@ -26,6 +26,7 @@ # #include +#include INTERFACE sff; @@ -33,3 +34,14 @@ device_t dev; device_t *i2c_bus; }; + +# +# Register upstream MAC driver to receive module/link event callbacks. +# ops: function pointer table (sfp_upstream_ops from sfp_fdt.h) +# arg: opaque context passed to each callback (typically MAC softc) +# +METHOD int register_upstream { + device_t dev; + const struct sfp_upstream_ops *ops; + void *arg; +}; diff --git a/sys/dev/sff/sfp_fdt.c b/sys/dev/sff/sfp_fdt.c index e566d8ced78c..fdedd84ba981 100644 --- a/sys/dev/sff/sfp_fdt.c +++ b/sys/dev/sff/sfp_fdt.c @@ -1,7 +1,8 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * - * Copyright © 2023 Dmitry Salychev + * Copyright (c) 2023 Dmitry Salychev + * Copyright (c) 2026 Mono Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,33 +27,1072 @@ */ /* - * Small Form Factor (SFF) Committee Pluggable (SFP) Transceiver (FDT-based). + * SFP/SFP+ transceiver management driver (FDT-based). + * + * Implements a state machine for SFP module hot-plug, EEPROM identification, + * embedded PHY management (RollBall and standard I2C-MDIO), and link + * monitoring. MAC drivers register upstream callbacks to receive module + * insert/remove and link up/down events. + * + * Inspired by Linux drivers/net/phy/sfp.c but simplified to a single + * state machine without phylink dependency. */ #include #include #include #include +#include +#include +#include +#include #include #include #include +#include +#include +#include +#include + +#include #include "sff_if.h" +/* State machine states */ +#define SFP_ST_EMPTY 0 /* No module */ +#define SFP_ST_PROBE 1 /* Module detected, reading EEPROM */ +#define SFP_ST_PRESENT 2 /* EEPROM read, classifying */ +#define SFP_ST_PHY_PROBE 3 /* Probing embedded PHY (copper) */ +#define SFP_ST_PHY_AN 4 /* PHY found, polling link */ +#define SFP_ST_DDM_POLL 5 /* No PHY, polling DDM RX_LOS */ +#define SFP_ST_WAIT_LOS 6 /* Fiber/DAC: monitoring LOS GPIO */ +#define SFP_ST_LINK_UP 7 /* Link established */ + +/* IEEE 802.3 Clause 45 MMD device numbers */ +#define MDIO_MMD_PMAPMD 1 +#define MDIO_MMD_AN 7 + +/* Standard C45 register offsets */ +#define MDIO_CTRL1 0 +#define MDIO_STAT1 1 +#define MDIO_DEVID1 2 +#define MDIO_DEVID2 3 +#define MDIO_PMA_EXTABLE 11 +#define MDIO_AN_10GBT_CTRL 32 +#define MDIO_AN_10GBT_STAT 33 + +/* STAT1 bits */ +#define MDIO_STAT1_LSTATUS (1 << 2) + +/* AN CTRL1 bits */ +#define MDIO_AN_CTRL1_XNP (1 << 13) +#define MDIO_AN_CTRL1_ENABLE (1 << 12) +#define MDIO_AN_CTRL1_RESTART (1 << 9) + +/* AN STAT1 bits */ +#define MDIO_AN_STAT1_COMPLETE (1 << 5) + +/* PMA EXTABLE bits */ +#define MDIO_PMA_EXTABLE_10GBT (1 << 2) +#define MDIO_PMA_EXTABLE_1000BT (1 << 5) +#define MDIO_PMA_EXTABLE_NBT (1 << 14) + +/* AN 10GBT_CTRL advertisement bits */ +#define MDIO_AN_10GBT_CTRL_ADV10G (1 << 12) +#define MDIO_AN_10GBT_CTRL_ADV5G (1 << 8) +#define MDIO_AN_10GBT_CTRL_ADV2_5G (1 << 7) + +/* AN 10GBT_STAT link partner bits */ +#define MDIO_AN_10GBT_STAT_LP10G (1 << 11) +#define MDIO_AN_10GBT_STAT_LP5G (1 << 6) +#define MDIO_AN_10GBT_STAT_LP2_5G (1 << 5) + +/* Standard I2C-MDIO bridge (SFF-8472) */ +#define MDIOI2C_I2C_ADDR 0x56 + +/* RollBall I2C-to-MDIO bridge */ +#define ROLLBALL_I2C_ADDR 0x51 +#define ROLLBALL_PAGE_REG 0x7F +#define ROLLBALL_PAGE 3 +#define ROLLBALL_PASSWORD_REG 0x7B +#define ROLLBALL_CMD_ADDR 0x80 +#define ROLLBALL_DATA_ADDR 0x81 +#define ROLLBALL_CMD_READ 0x02 +#define ROLLBALL_CMD_WRITE 0x01 +#define ROLLBALL_CMD_DONE 0x04 +#define ROLLBALL_POLL_RETRIES 20 +#define ROLLBALL_POLL_MS 20 + +/* PHY access protocol */ +#define SFP_PHY_ROLLBALL 1 +#define SFP_PHY_MDIOI2C 2 + +/* PHY boot retries (500ms each, up to 4.5s total) */ +#define SFP_PHY_BOOT_RETRIES 9 +#define SFP_PHY_BOOT_MS 500 + struct sfp_fdt_softc { - phandle_t ofw_node; - phandle_t i2c_bus; - - phandle_t mod_def; - phandle_t los; - phandle_t tx_fault; - phandle_t tx_disable; - phandle_t rx_rate; - phandle_t tx_rate; - uint32_t max_power; /* in mW */ + device_t sc_dev; + phandle_t sc_node; + + /* I2C */ + phandle_t sc_i2c_phandle; + device_t sc_i2c; /* resolved iicbus device */ + + /* GPIOs */ + gpio_pin_t sc_moddef0; /* module detect (active low) */ + gpio_pin_t sc_los; /* loss of signal */ + gpio_pin_t sc_txdis; /* TX disable (output) */ + + /* State machine */ + struct mtx sc_mtx; + int sc_state; + uint8_t sc_id[64]; /* EEPROM A0h base ID cache */ + uint8_t sc_connector; + + /* PHY (copper modules) */ + bool sc_has_phy; + int sc_phy_proto; + uint32_t sc_phy_id; + bool sc_phy_link; + int sc_phy_speed; + int sc_phy_retries; + + /* Fiber/DAC LOS tracking */ + bool sc_los_prev; + + /* Upstream (MAC driver) */ + const struct sfp_upstream_ops *sc_upstream_ops; + void *sc_upstream_arg; + + /* Polling */ + struct task sc_sm_task; + struct callout sc_poll; }; +#define SFP_LOCK(sc) mtx_lock(&(sc)->sc_mtx) +#define SFP_UNLOCK(sc) mtx_unlock(&(sc)->sc_mtx) + +/* Forward declarations */ +static void sfp_fdt_sm_task(void *arg, int pending); +static void sfp_fdt_poll_callout(void *arg); + +/* + * Lazily resolve I2C bus device. The I2C mux (pca954x) may attach + * after sfp_fdt, so we defer resolution until first use. + */ +static bool +sfp_fdt_resolve_i2c(struct sfp_fdt_softc *sc) +{ + device_t xdev; + + if (sc->sc_i2c != NULL) + return (true); + + xdev = OF_device_from_xref(OF_xref_from_node(sc->sc_i2c_phandle)); + if (xdev == NULL) + return (false); + + sc->sc_i2c = xdev; + return (true); +} + +/* + * ============================================================ + * I2C helpers + * ============================================================ + */ + +static int +sfp_fdt_i2c_read(struct sfp_fdt_softc *sc, uint8_t addr, uint8_t offset, + uint8_t *buf, int len) +{ + struct iic_msg msgs[2]; + int error; + + msgs[0].slave = addr << 1; + msgs[0].flags = IIC_M_WR; + msgs[0].len = 1; + msgs[0].buf = &offset; + msgs[1].slave = addr << 1; + msgs[1].flags = IIC_M_RD; + msgs[1].len = len; + msgs[1].buf = buf; + + error = iicbus_request_bus(sc->sc_i2c, sc->sc_dev, IIC_INTRWAIT); + if (error != 0) + return (error); + error = iicbus_transfer(sc->sc_i2c, msgs, 2); + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + + return (error); +} + +static int +sfp_fdt_i2c_write_raw(struct sfp_fdt_softc *sc, uint8_t addr, + uint8_t *buf, int len) +{ + struct iic_msg msg; + + msg.slave = addr << 1; + msg.flags = IIC_M_WR; + msg.len = len; + msg.buf = buf; + + return (iicbus_transfer(sc->sc_i2c, &msg, 1)); +} + +static int +sfp_fdt_i2c_read_raw(struct sfp_fdt_softc *sc, uint8_t addr, + uint8_t reg, uint8_t *buf, int len) +{ + struct iic_msg msgs[2]; + + msgs[0].slave = addr << 1; + msgs[0].flags = IIC_M_WR; + msgs[0].len = 1; + msgs[0].buf = ® + msgs[1].slave = addr << 1; + msgs[1].flags = IIC_M_RD; + msgs[1].len = len; + msgs[1].buf = buf; + + return (iicbus_transfer(sc->sc_i2c, msgs, 2)); +} + +/* + * ============================================================ + * EEPROM + * ============================================================ + */ + +static int +sfp_fdt_read_eeprom(struct sfp_fdt_softc *sc) +{ + int error; + + error = sfp_fdt_i2c_read(sc, SFP_EEPROM_ADDR, 0, + sc->sc_id, sizeof(sc->sc_id)); + if (error != 0) { + device_printf(sc->sc_dev, "SFP EEPROM read failed: %d\n", + error); + memset(sc->sc_id, 0, sizeof(sc->sc_id)); + } + return (error); +} + +static void +sfp_fdt_trim(char *dst, const uint8_t *src, int len) +{ + int i; + + memcpy(dst, src, len); + dst[len] = '\0'; + for (i = len - 1; i >= 0 && dst[i] == ' '; i--) + dst[i] = '\0'; +} + +static void +sfp_fdt_log_module(struct sfp_fdt_softc *sc) +{ + char vendor[SFP_VENDOR_LEN + 1]; + char partnum[SFP_PARTNUM_LEN + 1]; + uint8_t connector; + + sfp_fdt_trim(vendor, &sc->sc_id[SFP_VENDOR_OFFSET], SFP_VENDOR_LEN); + sfp_fdt_trim(partnum, &sc->sc_id[SFP_PARTNUM_OFFSET], SFP_PARTNUM_LEN); + connector = sc->sc_id[SFP_CONNECTOR_OFFSET]; + + device_printf(sc->sc_dev, + "SFP+ module: %s %s (connector 0x%02x%s)\n", + vendor, partnum, connector, + connector == SFP_CONNECTOR_RJ45 ? " RJ45" : ""); +} + +/* + * ============================================================ + * Standard I2C-MDIO (SFF-8472, address 0x56) + * ============================================================ + */ + +static int +mdioi2c_read(struct sfp_fdt_softc *sc, int devad, int reg) +{ + uint8_t addr[3], data[2]; + int error; + + addr[0] = 0x20 | (devad & 0x1F); + addr[1] = (reg >> 8) & 0xFF; + addr[2] = reg & 0xFF; + + error = iicbus_request_bus(sc->sc_i2c, sc->sc_dev, IIC_INTRWAIT); + if (error != 0) + return (-1); + + error = sfp_fdt_i2c_write_raw(sc, MDIOI2C_I2C_ADDR, addr, 3); + if (error == 0) { + struct iic_msg msg; + msg.slave = MDIOI2C_I2C_ADDR << 1; + msg.flags = IIC_M_RD; + msg.len = 2; + msg.buf = data; + error = iicbus_transfer(sc->sc_i2c, &msg, 1); + } + + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + + if (error != 0) + return (-1); + return ((data[0] << 8) | data[1]); +} + +static int +mdioi2c_write(struct sfp_fdt_softc *sc, int devad, int reg, int val) +{ + uint8_t buf[5]; + int error; + + buf[0] = devad & 0x1F; + buf[1] = (reg >> 8) & 0xFF; + buf[2] = reg & 0xFF; + buf[3] = (val >> 8) & 0xFF; + buf[4] = val & 0xFF; + + error = iicbus_request_bus(sc->sc_i2c, sc->sc_dev, IIC_INTRWAIT); + if (error != 0) + return (error); + error = sfp_fdt_i2c_write_raw(sc, MDIOI2C_I2C_ADDR, buf, 5); + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + + return (error); +} + +/* + * ============================================================ + * RollBall I2C-to-MDIO bridge (address 0x51) + * ============================================================ + */ + +/* Bus must already be acquired */ +static int +rollball_page_set(struct sfp_fdt_softc *sc, uint8_t *saved_page) +{ + uint8_t buf[2]; + int error; + + error = sfp_fdt_i2c_read_raw(sc, ROLLBALL_I2C_ADDR, + ROLLBALL_PAGE_REG, saved_page, 1); + if (error != 0) + return (error); + + buf[0] = ROLLBALL_PAGE_REG; + buf[1] = ROLLBALL_PAGE; + return (sfp_fdt_i2c_write_raw(sc, ROLLBALL_I2C_ADDR, buf, 2)); +} + +/* Bus must already be acquired */ +static int +rollball_page_restore(struct sfp_fdt_softc *sc, uint8_t saved_page) +{ + uint8_t buf[2] = { ROLLBALL_PAGE_REG, saved_page }; + + return (sfp_fdt_i2c_write_raw(sc, ROLLBALL_I2C_ADDR, buf, 2)); +} + +static int +rollball_init(struct sfp_fdt_softc *sc) +{ + uint8_t pw[] = { ROLLBALL_PASSWORD_REG, 0xFF, 0xFF, 0xFF, 0xFF }; + int error; + + error = iicbus_request_bus(sc->sc_i2c, sc->sc_dev, IIC_INTRWAIT); + if (error != 0) + return (error); + error = sfp_fdt_i2c_write_raw(sc, ROLLBALL_I2C_ADDR, pw, sizeof(pw)); + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + + return (error); +} + +static int +rollball_phy_read(struct sfp_fdt_softc *sc, int devad, int reg) +{ + uint8_t data_buf[4], cmd_buf[2], res[6]; + uint8_t saved_page; + int error, i; + + error = iicbus_request_bus(sc->sc_i2c, sc->sc_dev, IIC_INTRWAIT); + if (error != 0) + return (-1); + + error = rollball_page_set(sc, &saved_page); + if (error != 0) + goto out; + + /* Write devad + register address */ + data_buf[0] = ROLLBALL_DATA_ADDR; + data_buf[1] = devad; + data_buf[2] = (reg >> 8) & 0xFF; + data_buf[3] = reg & 0xFF; + error = sfp_fdt_i2c_write_raw(sc, ROLLBALL_I2C_ADDR, + data_buf, 4); + if (error != 0) + goto restore; + + /* Write read command */ + cmd_buf[0] = ROLLBALL_CMD_ADDR; + cmd_buf[1] = ROLLBALL_CMD_READ; + error = sfp_fdt_i2c_write_raw(sc, ROLLBALL_I2C_ADDR, + cmd_buf, 2); + if (error != 0) + goto restore; + + /* Restore page before polling */ + rollball_page_restore(sc, saved_page); + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + + /* Poll for completion */ + for (i = 0; i < ROLLBALL_POLL_RETRIES; i++) { + pause_sbt("rbpoll", SBT_1MS * ROLLBALL_POLL_MS, 0, C_PREL(2)); + + error = iicbus_request_bus(sc->sc_i2c, sc->sc_dev, + IIC_INTRWAIT); + if (error != 0) + return (-1); + + error = rollball_page_set(sc, &saved_page); + if (error != 0) { + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + return (-1); + } + + error = sfp_fdt_i2c_read_raw(sc, ROLLBALL_I2C_ADDR, + ROLLBALL_CMD_ADDR, res, 6); + + rollball_page_restore(sc, saved_page); + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + + if (error != 0) + return (-1); + + if (res[0] == ROLLBALL_CMD_DONE) + return ((res[4] << 8) | res[5]); + } + + return (-1); /* timeout */ + +restore: + rollball_page_restore(sc, saved_page); +out: + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + return (-1); +} + +static int +rollball_phy_write(struct sfp_fdt_softc *sc, int devad, int reg, int val) +{ + uint8_t data_buf[6], cmd_buf[2], status; + uint8_t saved_page; + int error, i; + + error = iicbus_request_bus(sc->sc_i2c, sc->sc_dev, IIC_INTRWAIT); + if (error != 0) + return (error); + + error = rollball_page_set(sc, &saved_page); + if (error != 0) + goto out; + + data_buf[0] = ROLLBALL_DATA_ADDR; + data_buf[1] = devad; + data_buf[2] = (reg >> 8) & 0xFF; + data_buf[3] = reg & 0xFF; + data_buf[4] = (val >> 8) & 0xFF; + data_buf[5] = val & 0xFF; + error = sfp_fdt_i2c_write_raw(sc, ROLLBALL_I2C_ADDR, + data_buf, 6); + if (error != 0) + goto restore; + + cmd_buf[0] = ROLLBALL_CMD_ADDR; + cmd_buf[1] = ROLLBALL_CMD_WRITE; + error = sfp_fdt_i2c_write_raw(sc, ROLLBALL_I2C_ADDR, + cmd_buf, 2); + if (error != 0) + goto restore; + + rollball_page_restore(sc, saved_page); + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + + /* Poll for completion */ + for (i = 0; i < ROLLBALL_POLL_RETRIES; i++) { + pause_sbt("rbpoll", SBT_1MS * ROLLBALL_POLL_MS, 0, C_PREL(2)); + + error = iicbus_request_bus(sc->sc_i2c, sc->sc_dev, + IIC_INTRWAIT); + if (error != 0) + return (error); + + error = rollball_page_set(sc, &saved_page); + if (error != 0) { + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + return (error); + } + + error = sfp_fdt_i2c_read_raw(sc, ROLLBALL_I2C_ADDR, + ROLLBALL_CMD_ADDR, &status, 1); + + rollball_page_restore(sc, saved_page); + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + + if (error != 0) + return (error); + + if (status == ROLLBALL_CMD_DONE) + return (0); + } + + return (ETIMEDOUT); + +restore: + rollball_page_restore(sc, saved_page); +out: + iicbus_release_bus(sc->sc_i2c, sc->sc_dev); + return (error); +} + +/* + * ============================================================ + * PHY register access — protocol dispatch + * ============================================================ + */ + +static int +sfp_phy_read(struct sfp_fdt_softc *sc, int devad, int reg) +{ + + if (sc->sc_phy_proto == SFP_PHY_MDIOI2C) + return (mdioi2c_read(sc, devad, reg)); + return (rollball_phy_read(sc, devad, reg)); +} + +static int +sfp_phy_write(struct sfp_fdt_softc *sc, int devad, int reg, int val) +{ + + if (sc->sc_phy_proto == SFP_PHY_MDIOI2C) + return (mdioi2c_write(sc, devad, reg, val)); + return (rollball_phy_write(sc, devad, reg, val)); +} + +/* + * ============================================================ + * PHY auto-negotiation and speed detection + * ============================================================ + */ + +static void +sfp_phy_config_aneg(struct sfp_fdt_softc *sc) +{ + int adv10g; + + /* Advertise 10G/5G/2.5G in register 7.32 */ + adv10g = MDIO_AN_10GBT_CTRL_ADV10G | + MDIO_AN_10GBT_CTRL_ADV5G | MDIO_AN_10GBT_CTRL_ADV2_5G; + sfp_phy_write(sc, MDIO_MMD_AN, MDIO_AN_10GBT_CTRL, adv10g); + + /* Enable and restart AN with XNP (Extended Next Page). + * 10GBASE-T capability is exchanged via extended next pages — + * without XNP, only base-page speeds (1G and below) negotiate. */ + sfp_phy_write(sc, MDIO_MMD_AN, MDIO_CTRL1, + MDIO_AN_CTRL1_XNP | MDIO_AN_CTRL1_ENABLE | + MDIO_AN_CTRL1_RESTART); + + device_printf(sc->sc_dev, "SFP+ PHY: auto-negotiation started\n"); +} + +static int +sfp_phy_read_speed(struct sfp_fdt_softc *sc) +{ + int stat, lpa10g, ctrl1; + + stat = sfp_phy_read(sc, MDIO_MMD_AN, MDIO_STAT1); + if (stat < 0 || !(stat & MDIO_AN_STAT1_COMPLETE)) + return (0); + + lpa10g = sfp_phy_read(sc, MDIO_MMD_AN, MDIO_AN_10GBT_STAT); + if (lpa10g >= 0) { + if (lpa10g & MDIO_AN_10GBT_STAT_LP10G) + return (10000); + if (lpa10g & MDIO_AN_10GBT_STAT_LP5G) + return (5000); + if (lpa10g & MDIO_AN_10GBT_STAT_LP2_5G) + return (2500); + } + + /* PMA/PMD CTRL1 speed select: bit 13=10G, bit 6=1G */ + ctrl1 = sfp_phy_read(sc, MDIO_MMD_PMAPMD, MDIO_CTRL1); + if (ctrl1 >= 0) { + if (ctrl1 & (1 << 13)) + return (10000); + if (ctrl1 & (1 << 6)) + return (1000); + return (100); + } + + return (0); +} + +/* + * ============================================================ + * Upstream callback helpers + * ============================================================ + */ + +static void +sfp_fdt_notify_insert(struct sfp_fdt_softc *sc) +{ + + if (sc->sc_upstream_ops != NULL && + sc->sc_upstream_ops->module_insert != NULL) + sc->sc_upstream_ops->module_insert(sc->sc_upstream_arg, + sc->sc_id, sizeof(sc->sc_id)); +} + +static void +sfp_fdt_notify_remove(struct sfp_fdt_softc *sc) +{ + + if (sc->sc_upstream_ops != NULL && + sc->sc_upstream_ops->module_remove != NULL) + sc->sc_upstream_ops->module_remove(sc->sc_upstream_arg); +} + +static void +sfp_fdt_notify_link_up(struct sfp_fdt_softc *sc, int speed) +{ + + if (sc->sc_upstream_ops != NULL && + sc->sc_upstream_ops->link_up != NULL) + sc->sc_upstream_ops->link_up(sc->sc_upstream_arg, speed); +} + +static void +sfp_fdt_notify_link_down(struct sfp_fdt_softc *sc) +{ + + if (sc->sc_upstream_ops != NULL && + sc->sc_upstream_ops->link_down != NULL) + sc->sc_upstream_ops->link_down(sc->sc_upstream_arg); +} + +/* + * ============================================================ + * State machine transitions + * ============================================================ + */ + +/* EMPTY → PROBE: module just detected */ +static void +sfp_fdt_sm_insert(struct sfp_fdt_softc *sc) +{ + + device_printf(sc->sc_dev, "SFP+ module detected, probing...\n"); + + /* De-assert TX disable so module MCU can fully boot */ + if (sc->sc_txdis != NULL) + gpio_pin_set_active(sc->sc_txdis, false); + + sc->sc_state = SFP_ST_PROBE; +} + +/* PROBE: wait T_serial (300ms) then read EEPROM */ +static void +sfp_fdt_sm_probe(struct sfp_fdt_softc *sc) +{ + bool present; + + /* Ensure I2C bus is resolved (may have deferred from attach) */ + if (!sfp_fdt_resolve_i2c(sc)) { + device_printf(sc->sc_dev, + "SFP: I2C bus not available yet\n"); + sc->sc_state = SFP_ST_EMPTY; + if (sc->sc_txdis != NULL) + gpio_pin_set_active(sc->sc_txdis, true); + return; + } + + /* SFF-8472 T_serial: 300ms after power-up before EEPROM accessible */ + pause_sbt("sfpwait", SBT_1MS * 300, 0, C_PREL(2)); + + /* Re-check module presence after sleep */ + if (sc->sc_moddef0 != NULL) { + gpio_pin_is_active(sc->sc_moddef0, &present); + if (!present) { + sc->sc_state = SFP_ST_EMPTY; + return; + } + } + + if (sfp_fdt_read_eeprom(sc) != 0) { + /* EEPROM read failed — back to EMPTY */ + sc->sc_state = SFP_ST_EMPTY; + if (sc->sc_txdis != NULL) + gpio_pin_set_active(sc->sc_txdis, true); + return; + } + + sc->sc_connector = sc->sc_id[SFP_CONNECTOR_OFFSET]; + sfp_fdt_log_module(sc); + sfp_fdt_notify_insert(sc); + + sc->sc_state = SFP_ST_PRESENT; +} + +/* PRESENT: classify module → PHY_PROBE (copper) or WAIT_LOS (fiber) */ +static void +sfp_fdt_sm_present(struct sfp_fdt_softc *sc) +{ + + if (sc->sc_connector == SFP_CONNECTOR_RJ45) { + sc->sc_phy_retries = 0; + sc->sc_state = SFP_ST_PHY_PROBE; + } else { + /* Fiber or DAC — monitor LOS */ + sc->sc_los_prev = true; /* assume no signal initially */ + sc->sc_state = SFP_ST_WAIT_LOS; + } +} + +/* PHY_PROBE: probe embedded PHY via RollBall then I2C-MDIO */ +static void +sfp_fdt_sm_phy_probe(struct sfp_fdt_softc *sc) +{ + int id1, id2, retry; + uint32_t phy_id; + int extable; + + id1 = -1; + + /* Try RollBall first */ + sc->sc_phy_proto = SFP_PHY_ROLLBALL; + if (rollball_init(sc) == 0) { + for (retry = 0; retry < SFP_PHY_BOOT_RETRIES; retry++) { + pause_sbt("phyboot", SBT_1MS * SFP_PHY_BOOT_MS, + 0, C_PREL(2)); + + id1 = sfp_phy_read(sc, MDIO_MMD_PMAPMD, MDIO_DEVID1); + if (id1 > 0 && id1 != 0xFFFF) + break; + } + } + + /* Fallback to standard I2C-MDIO */ + if (id1 <= 0 || id1 == 0xFFFF) { + sc->sc_phy_proto = SFP_PHY_MDIOI2C; + id1 = sfp_phy_read(sc, MDIO_MMD_PMAPMD, MDIO_DEVID1); + } + + if (id1 <= 0 || id1 == 0xFFFF) { + device_printf(sc->sc_dev, + "SFP+ PHY: no C45 PHY detected " + "(tried RollBall and I2C-MDIO)\n"); + sc->sc_phy_proto = 0; + sc->sc_has_phy = false; + /* Fall back to DDM polling for copper without PHY */ + sc->sc_state = SFP_ST_DDM_POLL; + return; + } + + id2 = sfp_phy_read(sc, MDIO_MMD_PMAPMD, MDIO_DEVID2); + if (id2 < 0 || id2 == 0xFFFF) { + device_printf(sc->sc_dev, + "SFP+ PHY: DEVID1=0x%04x but DEVID2 read failed\n", + id1 & 0xFFFF); + sc->sc_phy_proto = 0; + sc->sc_has_phy = false; + sc->sc_state = SFP_ST_DDM_POLL; + return; + } + + phy_id = ((uint32_t)id1 << 16) | id2; + + device_printf(sc->sc_dev, + "SFP+ PHY detected via %s: ID %08x " + "(OUI %06x model %02x rev %01x)\n", + sc->sc_phy_proto == SFP_PHY_ROLLBALL ? "RollBall" : "I2C-MDIO", + phy_id, + (id1 << 6) | (id2 >> 10), + (id2 >> 4) & 0x3F, + id2 & 0x0F); + + extable = sfp_phy_read(sc, MDIO_MMD_PMAPMD, MDIO_PMA_EXTABLE); + if (extable >= 0) { + device_printf(sc->sc_dev, + "SFP+ PHY abilities: %s%s%s\n", + (extable & MDIO_PMA_EXTABLE_10GBT) ? "10GBASE-T " : "", + (extable & MDIO_PMA_EXTABLE_1000BT) ? "1000BASE-T " : "", + (extable & MDIO_PMA_EXTABLE_NBT) ? "NBASE-T" : ""); + } + + sc->sc_phy_id = phy_id; + sc->sc_has_phy = true; + sc->sc_phy_link = false; + sc->sc_phy_speed = 0; + + sfp_phy_config_aneg(sc); + + sc->sc_state = SFP_ST_PHY_AN; +} + +/* PHY_AN: poll PHY link status */ +static void +sfp_fdt_sm_phy_poll(struct sfp_fdt_softc *sc) +{ + int stat, speed; + bool link_up; + + if (!sc->sc_has_phy) + return; + + stat = sfp_phy_read(sc, MDIO_MMD_PMAPMD, MDIO_STAT1); + if (stat < 0) + return; + + link_up = (stat & MDIO_STAT1_LSTATUS) != 0; + + if (link_up != sc->sc_phy_link) { + sc->sc_phy_link = link_up; + + if (link_up) { + speed = sfp_phy_read_speed(sc); + sc->sc_phy_speed = speed; + if (speed > 0) + device_printf(sc->sc_dev, + "SFP+ PHY: link up at %d Mbps\n", speed); + else + device_printf(sc->sc_dev, + "SFP+ PHY: link up (speed unknown)\n"); + sfp_fdt_notify_link_up(sc, speed); + sc->sc_state = SFP_ST_LINK_UP; + } else { + sc->sc_phy_speed = 0; + device_printf(sc->sc_dev, "SFP+ PHY: link down\n"); + sfp_fdt_notify_link_down(sc); + } + } + + /* Keep polling for speed if link up but speed not yet known */ + if (link_up && sc->sc_phy_speed == 0) { + speed = sfp_phy_read_speed(sc); + if (speed > 0) { + sc->sc_phy_speed = speed; + device_printf(sc->sc_dev, + "SFP+ PHY: negotiated %d Mbps\n", speed); + } + } +} + +/* DDM_POLL: poll DDM RX_LOS for copper modules without PHY */ +static void +sfp_fdt_sm_ddm_poll(struct sfp_fdt_softc *sc) +{ + uint8_t ddm_status; + int error; + bool link_up; + + /* Read DDM status byte (A2h offset 0x6E, bit 1 = RX_LOS) */ + error = sfp_fdt_i2c_read(sc, SFP_DIAG_ADDR, 0x6E, + &ddm_status, 1); + if (error != 0) + return; + + link_up = (ddm_status & 0x02) == 0; + + if (link_up && sc->sc_state == SFP_ST_DDM_POLL) { + device_printf(sc->sc_dev, "SFP+ DDM: link up\n"); + sfp_fdt_notify_link_up(sc, 0); + sc->sc_state = SFP_ST_LINK_UP; + } +} + +/* WAIT_LOS: poll LOS GPIO for fiber/DAC modules */ +static void +sfp_fdt_sm_wait_los(struct sfp_fdt_softc *sc) +{ + bool los; + + if (sc->sc_los != NULL) { + gpio_pin_is_active(sc->sc_los, &los); + if (!los) { + /* LOS clear = signal present = link up */ + sc->sc_los_prev = false; + device_printf(sc->sc_dev, "SFP+ fiber: link up\n"); + sfp_fdt_notify_link_up(sc, 0); + sc->sc_state = SFP_ST_LINK_UP; + } + } else { + /* No LOS GPIO — assume link up when module present */ + sfp_fdt_notify_link_up(sc, 0); + sc->sc_state = SFP_ST_LINK_UP; + } +} + +/* LINK_UP: monitor for link loss */ +static void +sfp_fdt_sm_link_up(struct sfp_fdt_softc *sc) +{ + + if (sc->sc_has_phy) { + /* PHY-based: check PMA/PMD link status */ + int stat = sfp_phy_read(sc, MDIO_MMD_PMAPMD, MDIO_STAT1); + if (stat >= 0 && !(stat & MDIO_STAT1_LSTATUS)) { + sc->sc_phy_link = false; + sc->sc_phy_speed = 0; + device_printf(sc->sc_dev, "SFP+ PHY: link down\n"); + sfp_fdt_notify_link_down(sc); + sc->sc_state = SFP_ST_PHY_AN; + } else if (stat >= 0 && sc->sc_phy_speed == 0) { + /* Link up but speed unknown — keep trying */ + int speed = sfp_phy_read_speed(sc); + if (speed > 0) { + sc->sc_phy_speed = speed; + device_printf(sc->sc_dev, + "SFP+ PHY: negotiated %d Mbps\n", speed); + } + } + } else if (sc->sc_connector == SFP_CONNECTOR_RJ45) { + /* DDM-based copper: check RX_LOS */ + uint8_t ddm_status; + int error = sfp_fdt_i2c_read(sc, SFP_DIAG_ADDR, 0x6E, + &ddm_status, 1); + if (error == 0 && (ddm_status & 0x02)) { + device_printf(sc->sc_dev, "SFP+ DDM: link down\n"); + sfp_fdt_notify_link_down(sc); + sc->sc_state = SFP_ST_DDM_POLL; + } + } else if (sc->sc_los != NULL) { + /* Fiber/DAC: check LOS GPIO */ + bool los; + gpio_pin_is_active(sc->sc_los, &los); + if (los && !sc->sc_los_prev) { + sc->sc_los_prev = true; + device_printf(sc->sc_dev, "SFP+ fiber: link down\n"); + sfp_fdt_notify_link_down(sc); + sc->sc_state = SFP_ST_WAIT_LOS; + } else if (!los && sc->sc_los_prev) { + sc->sc_los_prev = false; + } + } +} + +/* Module removal — reset all state */ +static void +sfp_fdt_sm_remove(struct sfp_fdt_softc *sc) +{ + int prev_state; + + prev_state = sc->sc_state; + + /* Assert TX disable */ + if (sc->sc_txdis != NULL) + gpio_pin_set_active(sc->sc_txdis, true); + + /* Notify upstream if we had gotten past EEPROM read */ + if (prev_state >= SFP_ST_PRESENT) + sfp_fdt_notify_remove(sc); + + /* Notify link down if we were up */ + if (prev_state == SFP_ST_LINK_UP) + sfp_fdt_notify_link_down(sc); + + /* Reset state */ + sc->sc_state = SFP_ST_EMPTY; + sc->sc_connector = 0; + sc->sc_has_phy = false; + sc->sc_phy_proto = 0; + sc->sc_phy_id = 0; + sc->sc_phy_link = false; + sc->sc_phy_speed = 0; + sc->sc_los_prev = true; + memset(sc->sc_id, 0, sizeof(sc->sc_id)); + + device_printf(sc->sc_dev, "SFP+ module removed\n"); +} + +/* + * ============================================================ + * State machine task — runs in taskqueue_thread (sleepable) + * ============================================================ + */ + +static void +sfp_fdt_sm_task(void *arg, int pending) +{ + struct sfp_fdt_softc *sc = arg; + bool present; + + if (sc->sc_moddef0 == NULL) + return; + + /* Check module presence */ + gpio_pin_is_active(sc->sc_moddef0, &present); + + /* Handle removal from any state */ + if (!present && sc->sc_state != SFP_ST_EMPTY) { + sfp_fdt_sm_remove(sc); + return; + } + + switch (sc->sc_state) { + case SFP_ST_EMPTY: + if (present) + sfp_fdt_sm_insert(sc); + break; + case SFP_ST_PROBE: + sfp_fdt_sm_probe(sc); + break; + case SFP_ST_PRESENT: + sfp_fdt_sm_present(sc); + break; + case SFP_ST_PHY_PROBE: + sfp_fdt_sm_phy_probe(sc); + break; + case SFP_ST_PHY_AN: + sfp_fdt_sm_phy_poll(sc); + break; + case SFP_ST_DDM_POLL: + sfp_fdt_sm_ddm_poll(sc); + break; + case SFP_ST_WAIT_LOS: + sfp_fdt_sm_wait_los(sc); + break; + case SFP_ST_LINK_UP: + sfp_fdt_sm_link_up(sc); + break; + } +} + +/* + * 1Hz callout — enqueues state machine task to taskqueue_thread + * so all I2C and GPIO work happens in sleepable context. + */ +static void +sfp_fdt_poll_callout(void *arg) +{ + struct sfp_fdt_softc *sc = arg; + + taskqueue_enqueue(taskqueue_thread, &sc->sc_sm_task); + callout_reset(&sc->sc_poll, hz, sfp_fdt_poll_callout, sc); +} + +/* + * ============================================================ + * Device methods + * ============================================================ + */ + static int sfp_fdt_probe(device_t dev) { @@ -65,11 +1105,8 @@ sfp_fdt_probe(device_t dev) s = device_get_property(dev, "i2c-bus", &node, sizeof(node), DEVICE_PROP_HANDLE); - if (s == -1) { - device_printf(dev, "%s: '%s' has no 'i2c-bus' property, s %zd\n", - __func__, ofw_bus_get_name(dev), s); + if (s == -1) return (ENXIO); - } device_set_desc(dev, "Small Form-factor Pluggable Transceiver"); return (BUS_PROBE_DEFAULT); @@ -79,58 +1116,123 @@ static int sfp_fdt_attach(device_t dev) { struct sfp_fdt_softc *sc; + device_t xdev; ssize_t s; int error; sc = device_get_softc(dev); - sc->ofw_node = ofw_bus_get_node(dev); + sc->sc_dev = dev; + sc->sc_node = ofw_bus_get_node(dev); + sc->sc_state = SFP_ST_EMPTY; - s = device_get_property(dev, "i2c-bus", &sc->i2c_bus, - sizeof(sc->i2c_bus), DEVICE_PROP_HANDLE); + /* Get I2C bus phandle (resolved lazily on first use) */ + s = device_get_property(dev, "i2c-bus", &sc->sc_i2c_phandle, + sizeof(sc->sc_i2c_phandle), DEVICE_PROP_HANDLE); if (s == -1) { - device_printf(dev, "%s: cannot find 'i2c-bus' property: %zd\n", - __func__, s); + device_printf(dev, "cannot find 'i2c-bus' property\n"); return (ENXIO); } - /* Optional properties */ - (void)device_get_property(dev, "mod-def0-gpios", &sc->mod_def, - sizeof(sc->mod_def), DEVICE_PROP_HANDLE); - (void)device_get_property(dev, "los-gpios", &sc->los, sizeof(sc->los), - DEVICE_PROP_HANDLE); - (void)device_get_property(dev, "tx-fault-gpios", &sc->tx_fault, - sizeof(sc->tx_fault), DEVICE_PROP_HANDLE); - (void)device_get_property(dev, "tx-disable-gpios", &sc->tx_disable, - sizeof(sc->tx_disable), DEVICE_PROP_HANDLE); - (void)device_get_property(dev, "rate-select0-gpios", &sc->rx_rate, - sizeof(sc->rx_rate), DEVICE_PROP_HANDLE); - (void)device_get_property(dev, "rate-select1-gpios", &sc->tx_rate, - sizeof(sc->tx_rate), DEVICE_PROP_HANDLE); - (void)device_get_property(dev, "maximum-power-milliwatt", &sc->max_power, - sizeof(sc->max_power), DEVICE_PROP_UINT32); - - error = OF_device_register_xref(OF_xref_from_node(sc->ofw_node), dev); - if (error != 0) - device_printf(dev, "%s: failed to register xref %#x\n", - __func__, OF_xref_from_node(sc->ofw_node)); + /* Register xref FIRST so MAC drivers can find us even if + * I2C/GPIO aren't ready yet (they may attach later) */ + error = OF_device_register_xref(OF_xref_from_node(sc->sc_node), dev); + if (error != 0) { + device_printf(dev, "failed to register xref\n"); + return (error); + } - return (error); + /* Try to resolve I2C bus now; if not ready, resolve lazily */ + xdev = OF_device_from_xref(OF_xref_from_node(sc->sc_i2c_phandle)); + if (xdev != NULL) + sc->sc_i2c = xdev; + + /* Acquire GPIOs from DT (all optional) */ + gpio_pin_get_by_ofw_property(dev, sc->sc_node, + "mod-def0-gpios", &sc->sc_moddef0); + gpio_pin_get_by_ofw_property(dev, sc->sc_node, + "los-gpios", &sc->sc_los); + gpio_pin_get_by_ofw_property(dev, sc->sc_node, + "tx-disable-gpios", &sc->sc_txdis); + + /* TX disable must be output, initially asserted (no module) */ + if (sc->sc_txdis != NULL) { + gpio_pin_setflags(sc->sc_txdis, GPIO_PIN_OUTPUT); + gpio_pin_set_active(sc->sc_txdis, true); + } + + /* Init mutex, task, and callout */ + mtx_init(&sc->sc_mtx, device_get_nameunit(dev), + "SFP state", MTX_DEF); + TASK_INIT(&sc->sc_sm_task, 0, sfp_fdt_sm_task, sc); + callout_init(&sc->sc_poll, CALLOUT_MPSAFE); + + if (sc->sc_moddef0 != NULL) { + device_printf(dev, "SFP+ cage ready (GPIOs OK), " + "polling started\n"); + /* Start 1Hz poll */ + callout_reset(&sc->sc_poll, hz, sfp_fdt_poll_callout, sc); + } else { + device_printf(dev, "SFP: no mod-def0 GPIO, " + "hot-plug disabled\n"); + } + + return (0); +} + +static int +sfp_fdt_detach(device_t dev) +{ + struct sfp_fdt_softc *sc; + + sc = device_get_softc(dev); + + callout_drain(&sc->sc_poll); + taskqueue_drain(taskqueue_thread, &sc->sc_sm_task); + mtx_destroy(&sc->sc_mtx); + + if (sc->sc_txdis != NULL) + gpio_pin_release(sc->sc_txdis); + if (sc->sc_los != NULL) + gpio_pin_release(sc->sc_los); + if (sc->sc_moddef0 != NULL) + gpio_pin_release(sc->sc_moddef0); + + return (0); } static int sfp_fdt_get_i2c_bus(device_t dev, device_t *i2c_bus) { struct sfp_fdt_softc *sc; - device_t xdev; KASSERT(i2c_bus != NULL, ("%s: i2c_bus is NULL", __func__)); sc = device_get_softc(dev); - xdev = OF_device_from_xref(OF_xref_from_node(sc->i2c_bus)); - if (xdev == NULL) + if (!sfp_fdt_resolve_i2c(sc)) return (ENXIO); - *i2c_bus = xdev; + *i2c_bus = sc->sc_i2c; + return (0); +} + +static int +sfp_fdt_register_upstream(device_t dev, const struct sfp_upstream_ops *ops, + void *arg) +{ + struct sfp_fdt_softc *sc; + + sc = device_get_softc(dev); + + if (sc->sc_upstream_ops != NULL) { + device_printf(dev, "upstream already registered\n"); + return (EBUSY); + } + + sc->sc_upstream_ops = ops; + sc->sc_upstream_arg = arg; + + device_printf(dev, "upstream MAC driver registered\n"); + return (0); } @@ -138,9 +1240,11 @@ static device_method_t sfp_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, sfp_fdt_probe), DEVMETHOD(device_attach, sfp_fdt_attach), + DEVMETHOD(device_detach, sfp_fdt_detach), - /* SFF */ + /* SFF interface */ DEVMETHOD(sff_get_i2c_bus, sfp_fdt_get_i2c_bus), + DEVMETHOD(sff_register_upstream, sfp_fdt_register_upstream), DEVMETHOD_END }; diff --git a/sys/dev/sff/sfp_fdt.h b/sys/dev/sff/sfp_fdt.h new file mode 100644 index 000000000000..ae81257493d3 --- /dev/null +++ b/sys/dev/sff/sfp_fdt.h @@ -0,0 +1,78 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Mono Technologies Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _SFP_FDT_H_ +#define _SFP_FDT_H_ + +/* + * SFP upstream operations — callbacks from sfp_fdt to MAC driver. + * MAC driver registers these via SFF_REGISTER_UPSTREAM(). + * All callbacks run in taskqueue_thread context (sleepable). + */ +struct sfp_upstream_ops { + /* + * Module inserted. Called after EEPROM read succeeds. + * id points to 64-byte SFP EEPROM A0h base ID data. + * Return 0 to accept, non-zero to reject the module. + */ + int (*module_insert)(void *arg, const uint8_t *id, int id_len); + + /* + * Module removed. Clean up any per-module state. + */ + void (*module_remove)(void *arg); + + /* + * Link came up. speed is in Mbps (10000, 5000, 2500, 1000). + * For fiber/DAC modules speed is 0 (unknown/fixed). + */ + void (*link_up)(void *arg, int speed); + + /* + * Link went down. + */ + void (*link_down)(void *arg); +}; + +/* SFF-8024 connector types */ +#define SFP_CONNECTOR_LC 0x07 +#define SFP_CONNECTOR_RJ45 0x22 + +/* SFF-8472 EEPROM A0h field offsets */ +#define SFP_ID_OFFSET 0 +#define SFP_CONNECTOR_OFFSET 2 +#define SFP_VENDOR_OFFSET 20 +#define SFP_VENDOR_LEN 16 +#define SFP_PARTNUM_OFFSET 40 +#define SFP_PARTNUM_LEN 16 + +/* I2C addresses */ +#define SFP_EEPROM_ADDR 0x50 /* EEPROM A0h page (7-bit) */ +#define SFP_DIAG_ADDR 0x51 /* A2h / RollBall (7-bit) */ +#define SFP_PHY_ADDR 0x56 /* Standard I2C-MDIO (7-bit) */ + +#endif /* _SFP_FDT_H_ */ diff --git a/sys/dev/uart/uart_dev_ns8250.c b/sys/dev/uart/uart_dev_ns8250.c index ced3b44b5749..ac760b064b24 100644 --- a/sys/dev/uart/uart_dev_ns8250.c +++ b/sys/dev/uart/uart_dev_ns8250.c @@ -78,6 +78,30 @@ static int broken_txfifo = 0; SYSCTL_INT(_hw, OID_AUTO, broken_txfifo, CTLFLAG_RWTUN, &broken_txfifo, 0, "UART FIFO has QEMU emulation bug"); +/* + * To use early printf on NXP QorIQ (LS1046A, etc.), add to kernel config: + * + * options SOCDEV_PA=0x2000000 + * options EARLY_PRINTF=qoriq + * + * SOCDEV_PA must be 2MB-aligned (L2 block constraint in locore.S). + * The DUART is at SOCDEV_PA + 0x1c0500. + */ +#if CHECK_EARLY_PRINTF(qoriq) +extern uintptr_t socdev_va; +static void +uart_qoriq_early_putc(int c) +{ + volatile uint8_t *base = (volatile uint8_t *)(socdev_va + 0x1c0500); + int limit = 10000; + + while ((base[5] & 0x20) == 0 && --limit > 0) + continue; + base[0] = c; +} +early_putc_t *early_putc = uart_qoriq_early_putc; +#endif /* CHECK_EARLY_PRINTF(qoriq) */ + /* * Clear pending interrupts. THRE is cleared by reading IIR. Data * that may have been received gets lost here. diff --git a/sys/kern/kern_cons.c b/sys/kern/kern_cons.c index 2b28c51984e1..ecee592d2a73 100644 --- a/sys/kern/kern_cons.c +++ b/sys/kern/kern_cons.c @@ -62,7 +62,11 @@ #include #include #include +#include #include +#ifdef __aarch64__ +#include +#endif #include #include #include @@ -496,6 +500,94 @@ cngets(char *cp, size_t size, int visible) } } +static bool cn_ts_newline = true; +static bool cn_ts_emitting; + +/* + * Emit a raw string directly to all console hardware, bypassing cnputc + * to avoid recursion. Handles \n → \r\n translation. + */ +static void +cn_puts_raw(const char *s) +{ + struct cn_device *cnd; + struct consdev *cn; + + for (; *s != '\0'; s++) { + STAILQ_FOREACH(cnd, &cn_devlist, cnd_next) { + cn = cnd->cnd_cn; + if (!kdb_active || + !(cn->cn_flags & CN_FLAG_NODEBUG)) + cn->cn_ops->cn_putc(cn, *s); + } + } +} + +/* + * Get uptime in seconds + microseconds. On arm64, read the generic timer + * directly (cntvct_el0 / cntfrq_el0) so timestamps work from the very + * first instruction, before timecounters are initialized. + */ +static void +cn_get_uptime(long *sec, int *usec) +{ +#ifdef __aarch64__ + static uint64_t cn_ts_freq; + uint64_t cnt, s, rem; + + if (__predict_false(cn_ts_freq == 0)) + cn_ts_freq = READ_SPECIALREG(cntfrq_el0); + cnt = READ_SPECIALREG(cntvct_el0); + s = cnt / cn_ts_freq; + rem = cnt % cn_ts_freq; + *sec = (long)s; + *usec = (int)(rem * 1000000 / cn_ts_freq); +#else + struct timeval tv; + + microuptime(&tv); + *sec = (long)tv.tv_sec; + *usec = (int)tv.tv_usec; +#endif +} + +/* + * Format a colored timestamp prefix into buf: [ 0.000132] + * Brackets are default color, the number is green. + */ +static void +cn_format_timestamp(char *buf, size_t len) +{ + long sec; + int usec; + + cn_get_uptime(&sec, &usec); + snprintf(buf, len, + "\033[0m[\033[34m%5ld.%06d\033[0m] ", sec, usec); +} + +static void +cn_emit_timestamp(void) +{ + char buf[48]; + + cn_format_timestamp(buf, sizeof(buf)); + cn_puts_raw(buf); +} + +#ifdef EARLY_PRINTF +static void +cn_emit_timestamp_early(void) +{ + char buf[48]; + const char *s; + + cn_format_timestamp(buf, sizeof(buf)); + for (s = buf; *s != '\0'; s++) + early_putc(*s); +} +#endif + void cnputc(int c) { @@ -505,8 +597,17 @@ cnputc(int c) #ifdef EARLY_PRINTF if (early_putc != NULL) { - if (c == '\n') + if (cn_ts_newline && !cn_ts_emitting && + c != '\n' && c != '\r' && c != '\0') { + cn_ts_emitting = true; + cn_emit_timestamp_early(); + cn_ts_emitting = false; + cn_ts_newline = false; + } + if (c == '\n') { early_putc('\r'); + cn_ts_newline = true; + } early_putc(c); return; } @@ -514,6 +615,15 @@ cnputc(int c) if (cn_mute || c == '\0') return; + + if (cn_ts_newline && !cn_ts_emitting && + c != '\n' && c != '\r') { + cn_ts_emitting = true; + cn_emit_timestamp(); + cn_ts_emitting = false; + cn_ts_newline = false; + } + STAILQ_FOREACH(cnd, &cn_devlist, cnd_next) { cn = cnd->cnd_cn; if (!kdb_active || !(cn->cn_flags & CN_FLAG_NODEBUG)) { @@ -522,6 +632,10 @@ cnputc(int c) cn->cn_ops->cn_putc(cn, c); } } + + if (c == '\n') + cn_ts_newline = true; + if (console_pausing && c == '\n' && !kdb_active) { for (cp = console_pausestr; *cp != '\0'; cp++) cnputc(*cp); diff --git a/sys/modules/dtb/freescale/Makefile b/sys/modules/dtb/freescale/Makefile index 871c15330fe3..22712ad7bb3f 100644 --- a/sys/modules/dtb/freescale/Makefile +++ b/sys/modules/dtb/freescale/Makefile @@ -1,8 +1,44 @@ # All the dts files for Freescale (now NXP) systems we support. -DTS= freescale/fsl-ls1028a-rdb.dts \ +DTS= freescale/fsl-ls1012a-frdm.dts \ + freescale/fsl-ls1012a-frwy.dts \ + freescale/fsl-ls1012a-oxalis.dts \ + freescale/fsl-ls1012a-qds.dts \ + freescale/fsl-ls1012a-rdb.dts \ + freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts \ + freescale/fsl-ls1028a-kontron-sl28.dts \ + freescale/fsl-ls1028a-kontron-sl28-var1.dts \ + freescale/fsl-ls1028a-kontron-sl28-var2.dts \ + freescale/fsl-ls1028a-kontron-sl28-var3-ads2.dts \ + freescale/fsl-ls1028a-kontron-sl28-var4.dts \ + freescale/fsl-ls1028a-qds-13bb.dts \ + freescale/fsl-ls1028a-qds-65bb.dts \ + freescale/fsl-ls1028a-qds-7777.dts \ + freescale/fsl-ls1028a-qds-85bb.dts \ + freescale/fsl-ls1028a-qds-899b.dts \ + freescale/fsl-ls1028a-qds-9999.dts \ + freescale/fsl-ls1028a-qds.dts \ + freescale/fsl-ls1028a-rdb.dts \ + freescale/fsl-ls1043a-qds.dts \ + freescale/fsl-ls1043a-rdb.dts \ + freescale/fsl-ls1046a-frwy.dts \ + freescale/fsl-ls1046a-qds.dts \ + freescale/fsl-ls1046a-rdb.dts \ + freescale/fsl-ls1088a-qds.dts \ + freescale/fsl-ls1088a-rdb.dts \ + freescale/fsl-ls1088a-ten64.dts \ + freescale/fsl-ls2080a-qds.dts \ + freescale/fsl-ls2080a-rdb.dts \ + freescale/fsl-ls2080a-simu.dts \ + freescale/fsl-ls2081a-rdb.dts \ + freescale/fsl-ls2088a-qds.dts \ + freescale/fsl-ls2088a-rdb.dts \ + freescale/fsl-lx2160a-bluebox3.dts \ + freescale/fsl-lx2160a-bluebox3-rev-a.dts \ freescale/fsl-lx2160a-clearfog-cx.dts \ freescale/fsl-lx2160a-honeycomb.dts \ freescale/fsl-lx2160a-qds.dts \ - freescale/fsl-lx2160a-rdb.dts + freescale/fsl-lx2160a-rdb.dts \ + freescale/fsl-lx2162a-qds.dts \ + freescale/mono-gateway-dk.dts .include diff --git a/sys/modules/sff/Makefile b/sys/modules/sff/Makefile index 96832070de63..dd05a7ea0b16 100644 --- a/sys/modules/sff/Makefile +++ b/sys/modules/sff/Makefile @@ -6,8 +6,9 @@ SRCS+= sff_if.c sff_if.h SRCS+= bus_if.h device_if.h .if !empty(OPT_FDT) -SRCS+= sfp_fdt.c \ - ofw_bus_if.h +SRCS+= sfp_fdt.c sfp_fdt.h \ + ofw_bus_if.h \ + gpio_if.h iicbus_if.h .endif .include diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c index 0f4ec01fb392..bae570386f79 100644 --- a/sys/net/if_bridge.c +++ b/sys/net/if_bridge.c @@ -303,6 +303,10 @@ int bridge_rtable_prune_period = BRIDGE_RTABLE_PRUNE_PERIOD; VNET_DEFINE_STATIC(uma_zone_t, bridge_rtnode_zone); #define V_bridge_rtnode_zone VNET(bridge_rtnode_zone) +/* L2 flow offload hooks (set by auto_bridge.ko) */ +bridge_l2flow_hook_t bridge_l2flow_hook; +bridge_fdb_can_expire_hook_t bridge_fdb_can_expire_hook; + static int bridge_clone_create(struct if_clone *, char *, size_t, struct ifc_data *, struct ifnet **); static int bridge_clone_destroy(struct if_clone *, struct ifnet *, uint32_t); @@ -2528,6 +2532,9 @@ bridge_forward(struct bridge_softc *sc, struct bridge_iflist *sbif, return; } + if (bridge_l2flow_hook != NULL) + bridge_l2flow_hook(ifp, m, src_if, dst_if); + bridge_enqueue(sc, dst_if, m); return; @@ -3110,8 +3117,16 @@ bridge_rtage(struct bridge_softc *sc) CK_LIST_FOREACH_SAFE(brt, &sc->sc_rtlist, brt_list, nbrt) { if ((brt->brt_flags & IFBAF_TYPEMASK) == IFBAF_DYNAMIC) { - if (time_uptime >= brt->brt_expire) + if (time_uptime >= brt->brt_expire) { + if (bridge_fdb_can_expire_hook != NULL && + bridge_fdb_can_expire_hook(brt->brt_addr, + sc->sc_ifp) == 0) { + brt->brt_expire = time_uptime + + sc->sc_brttimeout; + continue; + } bridge_rtnode_destroy(sc, brt); + } } } } diff --git a/sys/net/if_bridgevar.h b/sys/net/if_bridgevar.h index 511ca753858d..72c6f4abbf32 100644 --- a/sys/net/if_bridgevar.h +++ b/sys/net/if_bridgevar.h @@ -325,6 +325,18 @@ struct ifbpstpconf { extern void (*bridge_dn_p)(struct mbuf *, struct ifnet *); extern bool (*bridge_member_ifaddrs_p)(void); +/* + * L2 flow offload hooks — set by auto_bridge.ko, called from + * bridge_forward() and bridge_rtage() respectively. + */ +typedef void (*bridge_l2flow_hook_t)(struct ifnet *bridge_ifp, + struct mbuf *m, struct ifnet *src_if, struct ifnet *dst_if); +typedef int (*bridge_fdb_can_expire_hook_t)(const uint8_t *mac, + struct ifnet *bridge_ifp); + +extern bridge_l2flow_hook_t bridge_l2flow_hook; +extern bridge_fdb_can_expire_hook_t bridge_fdb_can_expire_hook; + #endif /* _KERNEL */ #endif /* _NET_IF_BRIDGEVAR_H_ */ diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h index 0f6e2e240222..0bddfb490874 100644 --- a/sys/net/pfvar.h +++ b/sys/net/pfvar.h @@ -1198,6 +1198,28 @@ VNET_DECLARE(pfsync_defer_t *, pfsync_defer_ptr); #define V_pfsync_defer_ptr VNET(pfsync_defer_ptr) extern pfsync_detach_ifnet_t *pfsync_detach_ifnet_ptr; +/* + * pf_notify — offload state change notification (separate from pfsync). + * + * Lock context for callbacks: + * INSERT: PF_HASHROW_LOCK held, NET_EPOCH + * UPDATE: PF_RULES_RLOCK + PF_STATE_LOCK(s) held, NET_EPOCH + * DELETE: PF_HASHROW_LOCK held, NET_EPOCH + * + * Callbacks MUST NOT sleep, acquire PF_RULES_WLOCK, or re-enter the + * pf state machine. + */ +typedef void pfnotify_insert_state_t(struct pf_kstate *); +typedef void pfnotify_update_state_t(struct pf_kstate *); +typedef void pfnotify_delete_state_t(struct pf_kstate *); + +VNET_DECLARE(pfnotify_insert_state_t *, pfnotify_insert_state_ptr); +#define V_pfnotify_insert_state_ptr VNET(pfnotify_insert_state_ptr) +VNET_DECLARE(pfnotify_update_state_t *, pfnotify_update_state_ptr); +#define V_pfnotify_update_state_ptr VNET(pfnotify_update_state_ptr) +VNET_DECLARE(pfnotify_delete_state_t *, pfnotify_delete_state_ptr); +#define V_pfnotify_delete_state_ptr VNET(pfnotify_delete_state_ptr) + void pfsync_state_export_1301(struct pfsync_state_1301 *, struct pf_kstate *); void pfsync_state_export_1400(struct pfsync_state_1400 *, diff --git a/sys/netgraph/ng_pppoe.c b/sys/netgraph/ng_pppoe.c index 712efce31336..6ada666d0c13 100644 --- a/sys/netgraph/ng_pppoe.c +++ b/sys/netgraph/ng_pppoe.c @@ -111,6 +111,14 @@ static const struct ng_parse_type ng_pppoe_sts_state_type = { &ng_pppoe_sts_type_fields }; +/* Parse type for struct ngpppoe_session_info */ +static const struct ng_parse_struct_field ng_pppoe_session_info_type_fields[] + = NG_PPPOE_SESSION_INFO_TYPE_INFO; +static const struct ng_parse_type ng_pppoe_session_info_type = { + &ng_parse_struct_type, + &ng_pppoe_session_info_type_fields +}; + /* List of commands and how to convert arguments to/from ASCII */ static const struct ng_cmdlist ng_pppoe_cmds[] = { { @@ -204,6 +212,13 @@ static const struct ng_cmdlist ng_pppoe_cmds[] = { &ngpppoe_init_data_state_type, NULL }, + { + NGM_PPPOE_COOKIE, + NGM_PPPOE_GET_SESSION_INFO, + "pppoe_get_session_info", + NULL, + &ng_pppoe_session_info_type + }, { 0 } }; @@ -1146,6 +1161,102 @@ ng_pppoe_rcvmsg(node_p node, item_p item, hook_p lasthook) privp->max_payload.hdr.tag_len = htons(sizeof(uint16_t)); privp->max_payload.data = htons(*((uint16_t *)msg->data)); break; + case NGM_PPPOE_GET_SESSION_INFO: + { + struct ngpppoe_session_info *info; + + if (msg->header.arglen >= sizeof(*ourmsg)) { + /* + * Hook name provided — return info for + * that single session. + */ + ourmsg = (struct ngpppoe_init_data *)msg->data; + ourmsg->hook[sizeof(ourmsg->hook) - 1] = '\0'; + hook = ng_findhook(node, ourmsg->hook); + if (hook == NULL) + LEAVE(ENOENT); + sp = NG_HOOK_PRIVATE(hook); + if (sp == NULL) + LEAVE(EINVAL); + + NG_MKRESPONSE(resp, msg, sizeof(*info), + M_NOWAIT); + if (resp == NULL) + LEAVE(ENOMEM); + + info = (struct ngpppoe_session_info *)resp->data; + strlcpy(info->hook, NG_HOOK_NAME(hook), + sizeof(info->hook)); + info->session_id = sp->Session_ID; + bcopy(sp->pkt_hdr.eh.ether_dhost, + info->peer_mac, ETHER_ADDR_LEN); + info->state = sp->state; + } else { + /* + * No args — return array of all connected + * sessions. + */ + struct sess_con *ssp; + int count, i, bucket; + + /* Count connected sessions. */ + count = 0; + for (bucket = 0; bucket < SESSHASHSIZE; + bucket++) { + mtx_lock(&privp->sesshash[bucket].mtx); + LIST_FOREACH(ssp, + &privp->sesshash[bucket].head, + sessions) { + if (ssp->state == + PPPOE_CONNECTED || + ssp->state == + PPPOE_NEWCONNECTED) + count++; + } + mtx_unlock(&privp->sesshash[bucket].mtx); + } + + if (count == 0) + LEAVE(ENOENT); + + NG_MKRESPONSE(resp, msg, + count * sizeof(*info), M_NOWAIT); + if (resp == NULL) + LEAVE(ENOMEM); + + info = (struct ngpppoe_session_info *)resp->data; + i = 0; + for (bucket = 0; bucket < SESSHASHSIZE && + i < count; bucket++) { + mtx_lock(&privp->sesshash[bucket].mtx); + LIST_FOREACH(ssp, + &privp->sesshash[bucket].head, + sessions) { + if (i >= count) + break; + if (ssp->state != + PPPOE_CONNECTED && + ssp->state != + PPPOE_NEWCONNECTED) + continue; + strlcpy(info[i].hook, + NG_HOOK_NAME(ssp->hook), + sizeof(info[i].hook)); + info[i].session_id = + ssp->Session_ID; + bcopy(ssp->pkt_hdr.eh.ether_dhost, + info[i].peer_mac, + ETHER_ADDR_LEN); + info[i].state = ssp->state; + i++; + } + mtx_unlock(&privp->sesshash[bucket].mtx); + } + /* Trim response if fewer found on second pass */ + resp->header.arglen = i * sizeof(*info); + } + break; + } case NGM_PPPOE_SEND_HURL: { struct mbuf *m; diff --git a/sys/netgraph/ng_pppoe.h b/sys/netgraph/ng_pppoe.h index d17766dd113a..72e4d6667e0a 100644 --- a/sys/netgraph/ng_pppoe.h +++ b/sys/netgraph/ng_pppoe.h @@ -88,7 +88,8 @@ enum cmd { NGM_PPPOE_SEND_HURL = 16, /* Send PADM HURL message */ NGM_PPPOE_HURL = 17, /* HURL for informational purposes */ NGM_PPPOE_SEND_MOTM = 18, /* Send PADM MOTM message */ - NGM_PPPOE_MOTM = 19 /* MOTM for informational purposes */ + NGM_PPPOE_MOTM = 19, /* MOTM for informational purposes */ + NGM_PPPOE_GET_SESSION_INFO = 20 /* Query session params for offload */ }; /*********************** @@ -281,4 +282,27 @@ struct maxptag { #define TAGS_HUNIQ 3 /* for PADT */ +/* + * Returned by NGM_PPPOE_GET_SESSION_INFO. + * + * If sent with a hook name (struct ngpppoe_init_data, data_len=0), + * returns info for that one session. If sent with arglen=0, + * returns an array of all connected sessions. + */ +struct ngpppoe_session_info { + char hook[NG_HOOKSIZ]; /* hook name */ + uint16_t session_id; /* PPPoE session ID (host order) */ + uint8_t peer_mac[ETHER_ADDR_LEN]; /* AC/peer MAC address */ + uint8_t state; /* enum state value */ + uint8_t _pad; +}; + +#define NG_PPPOE_SESSION_INFO_TYPE_INFO { \ + { "hook", &ng_parse_hookbuf_type }, \ + { "session_id", &ng_parse_uint16_type }, \ + { "peer_mac", &ng_parse_enaddr_type }, \ + { "state", &ng_parse_uint8_type }, \ + { NULL } \ +} + #endif /* _NETGRAPH_NG_PPPOE_H_ */ diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c index c9b61f7b6521..dac5ce76f336 100644 --- a/sys/netpfil/pf/pf.c +++ b/sys/netpfil/pf/pf.c @@ -1670,6 +1670,8 @@ pf_state_insert(struct pfi_kkif *kif, struct pfi_kkif *orig_kif, pf_counter_u64_add(&V_pf_status.fcounters[FCNT_STATE_INSERT], 1); if (V_pfsync_insert_state_ptr != NULL) V_pfsync_insert_state_ptr(s); + if (V_pfnotify_insert_state_ptr != NULL) + V_pfnotify_insert_state_ptr(s); /* Returns locked. */ return (0); @@ -2457,6 +2459,8 @@ pf_unlink_state(struct pf_kstate *s) if (V_pfsync_delete_state_ptr != NULL) V_pfsync_delete_state_ptr(s); + if (V_pfnotify_delete_state_ptr != NULL) + V_pfnotify_delete_state_ptr(s); STATE_DEC_COUNTERS(s); @@ -8652,6 +8656,8 @@ pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, if (s != NULL) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } @@ -8717,6 +8723,8 @@ pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, if (s != NULL) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } @@ -8750,6 +8758,8 @@ pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, if (action == PF_PASS) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } else if (s == NULL) { @@ -8770,6 +8780,8 @@ pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, if (action == PF_PASS) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } else if (s == NULL) @@ -8792,6 +8804,8 @@ pf_test(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, if (action == PF_PASS) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } else if (s == NULL) @@ -9310,6 +9324,8 @@ pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb if (action == PF_PASS) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } else if (s == NULL) { @@ -9374,6 +9390,8 @@ pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb if (action == PF_PASS) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } else if (s == NULL) @@ -9404,6 +9422,8 @@ pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb if (action == PF_PASS) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } else if (s == NULL) { @@ -9431,6 +9451,8 @@ pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb if (action == PF_PASS) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } else if (s == NULL) @@ -9444,6 +9466,8 @@ pf_test6(int dir, int pflags, struct ifnet *ifp, struct mbuf **m0, struct inpcb if (action == PF_PASS) { if (V_pfsync_update_state_ptr != NULL) V_pfsync_update_state_ptr(s); + if (V_pfnotify_update_state_ptr != NULL) + V_pfnotify_update_state_ptr(s); r = s->rule.ptr; a = s->anchor.ptr; } else if (s == NULL) diff --git a/sys/netpfil/pf/pf_ioctl.c b/sys/netpfil/pf/pf_ioctl.c index 261174aecf5b..fb2ecfd83df3 100644 --- a/sys/netpfil/pf/pf_ioctl.c +++ b/sys/netpfil/pf/pf_ioctl.c @@ -304,6 +304,11 @@ VNET_DEFINE(pfsync_clear_states_t *, pfsync_clear_states_ptr); VNET_DEFINE(pfsync_defer_t *, pfsync_defer_ptr); pfsync_detach_ifnet_t *pfsync_detach_ifnet_ptr; +/* pf_notify */ +VNET_DEFINE(pfnotify_insert_state_t *, pfnotify_insert_state_ptr); +VNET_DEFINE(pfnotify_update_state_t *, pfnotify_update_state_ptr); +VNET_DEFINE(pfnotify_delete_state_t *, pfnotify_delete_state_ptr); + /* pflog */ pflog_packet_t *pflog_packet_ptr = NULL; diff --git a/sys/sys/random.h b/sys/sys/random.h index aa4457fc8c22..6ff4f163f327 100644 --- a/sys/sys/random.h +++ b/sys/sys/random.h @@ -103,6 +103,7 @@ enum random_entropy_source { RANDOM_PURE_VMGENID, RANDOM_PURE_QUALCOMM, RANDOM_PURE_ARMV8, + RANDOM_PURE_CAAM, ENTROPYSOURCE }; _Static_assert(ENTROPYSOURCE <= 32, diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 973a7604fe5f..88cd70878b37 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -212,6 +212,17 @@ critical_exit(void) #ifdef EARLY_PRINTF typedef void early_putc_t(int ch); extern early_putc_t *early_putc; +#define CHECK_EARLY_PRINTF(x) \ + __CONCAT(early_printf_, EARLY_PRINTF) == __CONCAT(early_printf_, x) +#define early_printf_1 1 +#define early_printf_mvebu 2 +#define early_printf_ns8250 3 +#define early_printf_pl011 4 +#define early_printf_snps 5 +#define early_printf_sbi 6 +#define early_printf_qoriq 7 +#else +#define CHECK_EARLY_PRINTF(x) 0 #endif int kvprintf(char const *, void (*)(int, void*), void *, int, __va_list) __printflike(1, 0);