feat(controller): DualSense haptics PCM passthrough over USB audio

app/src/main/java/com/limelight/Game.kt

@@ -35,6 +35,7 @@ import com.limelight.binding.video.PerfOverlayListener
 import com.limelight.binding.video.PerformanceInfo
 import com.limelight.nvstream.NvConnection
 import com.limelight.nvstream.StreamConfiguration
+import com.limelight.nvstream.Ds5HapticsPcmFrame
 import com.limelight.nvstream.HdrModePolicy
 import com.limelight.nvstream.http.ComputerDetails
 import com.limelight.nvstream.http.AdaptiveBitrateService
@@ -2098,6 +2099,10 @@ class Game : Activity(), SurfaceHolder.Callback,
         controllerHandler.handleSetControllerLED(controllerNumber, r, g, b)
     }
 
+    override fun ds5HapticsPcm(frame: Ds5HapticsPcmFrame) {
+        controllerHandler.handleDs5HapticsPcm(frame)
+    }
+
     private fun prepareFramegenSurface(outputSurface: Surface, showEnabledToast: Boolean) {
         releaseFramegenCapture()
 

app/src/main/java/com/limelight/binding/input/ControllerHandler.kt

@@ -6,6 +6,9 @@ import android.content.Context
 import android.hardware.Sensor
 import android.hardware.SensorManager
 import android.hardware.input.InputManager
+import android.hardware.usb.UsbDeviceConnection
+import android.hardware.usb.UsbEndpoint
+import android.hardware.usb.UsbInterface
 import android.hardware.usb.UsbManager
 import android.os.Build
 import android.os.Handler
@@ -25,6 +28,8 @@ import com.limelight.binding.input.driver.AbstractController
 import com.limelight.binding.input.driver.UsbDriverListener
 import com.limelight.binding.input.driver.UsbDriverService
 import com.limelight.binding.input.haptics.ControllerHapticsCoordinator
+import com.limelight.binding.input.haptics.Ds5HapticsPump
+import com.limelight.nvstream.Ds5HapticsPcmFrame
 import com.limelight.nvstream.NvConnection
 import com.limelight.nvstream.input.ControllerPacket
 import com.limelight.nvstream.input.MouseButtonPacket
@@ -2724,6 +2729,48 @@ class ControllerHandler(
         }
     }
 
+    override fun reportControllerTouch(
+        controllerId: Int,
+        eventType: Byte,
+        pointerId: Int,
+        x: Float,
+        y: Float
+    ) {
+        val context = usbDeviceContexts[controllerId] ?: return
+        if (prefConfig.multiController && !context.assignedControllerNumber) return
+        if (context.shortcutState.isLocalInputCaptureActive()) return
+
+        // The Linux DS5 backend distinguishes contact/release via pressure.
+        val pressure = when (eventType) {
+            MoonBridge.LI_TOUCH_EVENT_DOWN, MoonBridge.LI_TOUCH_EVENT_MOVE -> 1f
+            else -> 0f
+        }
+        conn.sendControllerTouchEvent(
+            context.controllerNumber.toByte(), eventType, pointerId, x, y, pressure
+        )
+    }
+
+    override fun isUsbControllerReady(controllerId: Int): Boolean {
+        val context = usbDeviceContexts[controllerId] ?: return false
+        if (prefConfig.multiController && !context.assignedControllerNumber) return false
+        return context.controllerArrival.isReported
+    }
+
+    override fun onDs5AudioInterfaceAvailable(
+        controllerId: Int,
+        connection: UsbDeviceConnection,
+        streamingInterface: UsbInterface,
+        isoEndpoint: UsbEndpoint
+    ) {
+        val context = usbDeviceContexts[controllerId] ?: return
+        val pump = Ds5HapticsPump(connection, streamingInterface, isoEndpoint)
+        hapticsCoordinator.attachDs5HapticsPump(controllerId, context.controllerNumber, pump)
+    }
+
+    override fun onDs5AudioInterfaceGone(controllerId: Int) {
+        hapticsCoordinator.detachDs5HapticsPump(controllerId)
+    }
+
     // ========== Sensor Management ==========
 
     fun handleSetMotionEventState(controllerNumber: Short, motionType: Byte, reportRateHz: Short) {
@@ -2874,4 +2921,8 @@ class ControllerHandler(
 
     fun handleSetControllerLED(controllerNumber: Short, r: Byte, g: Byte, b: Byte) =
         rumbleManager.handleSetControllerLED(controllerNumber, r, g, b)
+
+    fun handleDs5HapticsPcm(frame: Ds5HapticsPcmFrame) {
+        hapticsCoordinator.submitDs5HapticsPcm(frame)
+    }
 }

app/src/main/java/com/limelight/binding/input/driver/AbstractController.kt

@@ -1,8 +1,8 @@
 package com.limelight.binding.input.driver
 
 abstract class AbstractController(
-    private val deviceId: Int,
-    private val listener: UsbDriverListener,
+    protected val deviceId: Int,
+    protected val listener: UsbDriverListener,
     private val vendorId: Int,
     private val productId: Int
 ) {
@@ -75,4 +75,10 @@ abstract class AbstractController(
     protected fun notifyBatteryState(batteryState: Byte, batteryPercentage: Byte) {
         listener.reportControllerBattery(deviceId, batteryState, batteryPercentage)
     }
+
+    protected fun isControllerReady(): Boolean = listener.isUsbControllerReady(deviceId)
+
+    protected fun notifyControllerTouch(eventType: Byte, pointerId: Int, x: Float, y: Float) {
+        listener.reportControllerTouch(deviceId, eventType, pointerId, x, y)
+    }
 }

app/src/main/java/com/limelight/binding/input/driver/AbstractDualSenseController.kt

@@ -34,6 +34,10 @@ abstract class AbstractDualSenseController(
     protected var inEndpt: UsbEndpoint? = null
     protected var outEndpt: UsbEndpoint? = null
 
+    // The UAC audioStreamingOut alt setting and its iso OUT endpoint, when the
+    // controller exposes the DualSense audio topology. Null on non-DS5 pads.
+    private var audioInterface: Pair<UsbInterface, UsbEndpoint>? = null
+
     // IMU data fields
     protected var gyroX = 0f
     protected var gyroY = 0f
@@ -153,17 +157,56 @@ abstract class AbstractDualSenseController(
             return false
         }
 
+        discoverAudioInterface()
+
         inputThread = createInputThread()
         inputThread!!.start()
         return true
     }
 
+    /**
+     * Discovers the UAC audio streaming OUT interface (the alt setting that
+     * carries the isochronous OUT endpoint) and notifies the listener so the
+     * haptics coordinator can create a PCM pump. The pump owns the alternate
+     * setting lifecycle; the interface itself was already claimed above.
+     */
+    private fun discoverAudioInterface() {
+        for (i in 0 until device.interfaceCount) {
+            val iface = device.getInterface(i)
+            if (iface.interfaceClass != UsbConstants.USB_CLASS_AUDIO ||
+                iface.interfaceSubclass != 0x02 // Audio Streaming
+            ) {
+                continue
+            }
+            // Alt 0 carries no endpoints; the alt 1 entry exposes the iso OUT
+            // endpoint (Android surfaces each alternate setting separately).
+            for (j in 0 until iface.endpointCount) {
+                val ep = iface.getEndpoint(j)
+                if (ep.direction == UsbConstants.USB_DIR_OUT &&
+                    ep.type == UsbConstants.USB_ENDPOINT_XFER_ISOC
+                ) {
+                    Log.i("DualSenseController", "UAC streaming OUT iface=${iface.id} ep=0x${Integer.toHexString(ep.address)}")
+                    audioInterface = iface to ep
+                    listener.onDs5AudioInterfaceAvailable(deviceId, connection, iface, ep)
+                    return
+                }
+            }
+        }
+    }
+
     override fun stop() {
         synchronized(this) {
             if (stopped) return
             stopped = true
         }
 
+        // Tear down the haptics pump first: it needs a live connection to
+        // restore the audio interface to alt 0 and release iso bandwidth.
+        if (audioInterface != null) {
+            audioInterface = null
+            listener.onDs5AudioInterfaceGone(deviceId)
+        }
+
         // Hold the output lock across the final clear so a report already queued by
         // the rumble worker cannot re-engage an effect after this point.
         synchronized(outputLock) {

app/src/main/java/com/limelight/binding/input/driver/DualSenseController.kt

@@ -18,10 +18,19 @@ class DualSenseController(
 
     override val supportsAdaptiveTriggers: Boolean = true
 
+    private class TouchSlot {
+        var down = false
+        var lastX = -1f
+        var lastY = -1f
+    }
+
+    private val touchSlots = Array(2) { TouchSlot() }
+
     init {
         capabilities = (capabilities.toInt() or
                 MoonBridge.LI_CCAP_BATTERY_STATE.toInt() or
-                MoonBridge.LI_CCAP_RGB_LED.toInt()).toShort()
+                MoonBridge.LI_CCAP_RGB_LED.toInt() or
+                MoonBridge.LI_CCAP_TOUCHPAD.toInt()).toShort()
     }
 
     private fun normalizeThumbStickAxis(value: Int): Float {
@@ -128,10 +137,59 @@ class DualSenseController(
         }
 
         reportBattery(buffer.get(BATTERY_OFFSET))
+        reportTouch(buffer, 0, TOUCH1_COUNTER_OFFSET, TOUCH1_DATA_OFFSET)
+        reportTouch(buffer, 1, TOUCH2_COUNTER_OFFSET, TOUCH2_DATA_OFFSET)
 
         return true
     }
 
+    private fun reportTouch(buffer: ByteBuffer, slotIndex: Int, counterOffset: Int, dataOffset: Int) {
+        val slot = touchSlots[slotIndex]
+
+        // Don't consume touch state until the controller has reported arrival and
+        // (in multi-controller mode) been assigned a number; otherwise the first
+        // DOWN would be dropped by the handler while the stationary finger never
+        // re-triggers an event.
+        if (!isControllerReady()) {
+            return
+        }
+
+        val down = (buffer.get(counterOffset).toInt() and 0x80) == 0
+
+        if (!down) {
+            if (slot.down) {
+                notifyControllerTouch(MoonBridge.LI_TOUCH_EVENT_UP, slotIndex, slot.lastX, slot.lastY)
+            }
+            slot.down = false
+            return
+        }
+
+        // Contact position: 12-bit X and Y, normalized by the touchpad panel size.
+        val d0 = buffer.get(dataOffset).toInt() and 0xFF
+        val d1 = buffer.get(dataOffset + 1).toInt() and 0xFF
+        val d2 = buffer.get(dataOffset + 2).toInt() and 0xFF
+        val x = (d0 or ((d1 and 0x0F) shl 8)) / TOUCHPAD_WIDTH
+        val y = ((d1 shr 4) or (d2 shl 4)) / TOUCHPAD_HEIGHT
+
+        if (!slot.down) {
+            notifyControllerTouch(MoonBridge.LI_TOUCH_EVENT_DOWN, slotIndex, x, y)
+        } else if (x != slot.lastX || y != slot.lastY) {
+            notifyControllerTouch(MoonBridge.LI_TOUCH_EVENT_MOVE, slotIndex, x, y)
+        }
+        slot.down = true
+        slot.lastX = x
+        slot.lastY = y
+    }
+
+    private fun releaseTouchContacts() {
+        touchSlots.forEachIndexed { index, slot ->
+            if (slot.down) {
+                notifyControllerTouch(MoonBridge.LI_TOUCH_EVENT_UP, index, slot.lastX, slot.lastY)
+                slot.down = false
+            }
+        }
+    }
+
     private fun reportBattery(batteryByte: Byte) {
         val status = (batteryByte.toInt() shr 4) and 0x0F
         val percentage = ((batteryByte.toInt() and 0x0F) * 10 + 5).coerceAtMost(100).toByte()
@@ -149,6 +207,12 @@ class DualSenseController(
         }
     }
 
+    override fun stop() {
+        // Release held touchpad contacts so the host doesn't keep a stuck finger.
+        releaseTouchContacts()
+        super.stop()
+    }
+
     override fun doInit(): Boolean {
         Log.d("DualSenseController", "doInit")
         sendCommand(getDualSenseInit())
@@ -231,6 +295,15 @@ class DualSenseController(
 
     companion object {
         private const val BATTERY_OFFSET = 53
+        // Offsets in the full 64-byte input report (report ID at index 0). SDL's
+        // PS5StatePacket_t comments exclude the report ID, so wire = struct + 1.
+        private const val TOUCH1_COUNTER_OFFSET = 33
+        private const val TOUCH1_DATA_OFFSET = 34
+        private const val TOUCH2_COUNTER_OFFSET = 37
+        private const val TOUCH2_DATA_OFFSET = 38
+        private const val TOUCHPAD_WIDTH = 1920f
+        // Matches the Linux hid-playstation DualSense ABS_MT range (0..1079).
+        private const val TOUCHPAD_HEIGHT = 1080f
         private val SUPPORTED_VENDORS = intArrayOf(0x054C, 0x1532)
         private val SUPPORTED_PRODUCTS = intArrayOf(0x0CE6, 0x0DF2, 0x100b, 0x100c)
 

app/src/main/java/com/limelight/binding/input/driver/UsbDriverListener.kt

@@ -1,5 +1,9 @@
 package com.limelight.binding.input.driver
 
+import android.hardware.usb.UsbDeviceConnection
+import android.hardware.usb.UsbEndpoint
+import android.hardware.usb.UsbInterface
+
 interface UsbDriverListener {
     fun reportControllerState(
         controllerId: Int, buttonFlags: Int,
@@ -16,4 +20,33 @@ interface UsbDriverListener {
 
     // Report battery state sourced from the USB controller itself
     fun reportControllerBattery(controllerId: Int, batteryState: Byte, batteryPercentage: Byte) {}
+
+    // Report touchpad touch events sourced from the USB controller itself
+    fun reportControllerTouch(
+        controllerId: Int,
+        eventType: Byte,
+        pointerId: Int,
+        x: Float,
+        y: Float
+    ) {}
+
+    // Whether the controller has reported arrival and received its controller
+    // number. Stateful consumers (e.g. touch tracking) must not consume state
+    // transitions until this is true.
+    fun isUsbControllerReady(controllerId: Int): Boolean = true
+
+    // Report that a DualSense UAC audio streaming interface is available.
+    // The handler should create a Ds5HapticsPump and attach it to the
+    // haptics coordinator.
+    fun onDs5AudioInterfaceAvailable(
+        controllerId: Int,
+        connection: UsbDeviceConnection,
+        streamingInterface: UsbInterface,
+        isoEndpoint: UsbEndpoint
+    ) {}
+
+    // Report that the previously announced DualSense audio interface is going
+    // away. The handler must stop and detach the pump before the connection
+    // closes.
+    fun onDs5AudioInterfaceGone(controllerId: Int) {}
 }