diff --git a/samples/gestures_heuristic/README.md b/samples/gestures_heuristic/README.md new file mode 100644 index 0000000..22e8cf5 --- /dev/null +++ b/samples/gestures_heuristic/README.md @@ -0,0 +1,9 @@ +# Heuristic Gesture HUD + +This sample enables the heuristic gesture recognizer and shows a simple HUD that +displays the most confident gesture per hand while logging gesture start/end +events to the console. Use it to validate pinch, open-palm, fist, thumbs-up, +point, and spread detections in the simulator or on device. + +Run `npm run serve` (or your preferred static server) and open +`samples/gestures_heuristic/` in a WebXR-capable browser. diff --git a/samples/gestures_heuristic/index.html b/samples/gestures_heuristic/index.html new file mode 100644 index 0000000..e2dc5b5 --- /dev/null +++ b/samples/gestures_heuristic/index.html @@ -0,0 +1,42 @@ + + + + Gestures: Heuristic HUD + + + + + + + + + + + + + + diff --git a/samples/gestures_heuristic/main.js b/samples/gestures_heuristic/main.js new file mode 100644 index 0000000..764ab87 --- /dev/null +++ b/samples/gestures_heuristic/main.js @@ -0,0 +1,212 @@ +import 'xrblocks/addons/simulator/SimulatorAddons.js'; + +import * as xb from 'xrblocks'; + +const options = new xb.Options(); +options.enableReticles(); +options.enableGestures(); + +options.gestures.setGestureEnabled('point', true); +options.gestures.setGestureEnabled('spread', true); + +options.hands.enabled = true; +options.hands.visualization = true; +options.hands.visualizeJoints = true; +options.hands.visualizeMeshes = true; + +options.simulator.defaultMode = xb.SimulatorMode.POSE; + +function createHudElement() { + const style = document.createElement('style'); + style.textContent = ` + #gesture-hud { + position: fixed; + top: 12px; + right: 12px; + min-width: 220px; + padding: 12px; + border-radius: 12px; + background: rgba(10, 12, 20, 0.82); + color: #f4f4f4; + font-family: 'Inter', 'Helvetica Neue', Arial, sans-serif; + font-size: 13px; + line-height: 1.4; + box-shadow: 0 6px 24px rgba(0, 0, 0, 0.3); + z-index: 9999; + } + #gesture-hud h2 { + margin: 0 0 8px; + font-size: 14px; + font-weight: 700; + letter-spacing: 0.01em; + } + #gesture-hud .hand-row { + display: flex; + align-items: center; + justify-content: space-between; + padding: 6px 8px; + border-radius: 8px; + background: rgba(255, 255, 255, 0.04); + margin-bottom: 6px; + } + #gesture-hud .hand-row:last-child { + margin-bottom: 0; + } + #gesture-hud .hand-label { + text-transform: uppercase; + letter-spacing: 0.08em; + font-size: 11px; + opacity: 0.75; + } + #gesture-hud .gesture { + font-weight: 700; + } + #gesture-hud .gesture[data-active="false"] { + opacity: 0.65; + } + `; + document.head.appendChild(style); + + const container = document.createElement('div'); + container.id = 'gesture-hud'; + container.innerHTML = ` +

Gestures

+
+ Left + None +
+
+ Right + None +
+ `; + document.body.appendChild(container); + return container; +} + +class GestureLogger extends xb.Script { + init() { + const gestures = xb.core.gestureRecognition; + if (!gestures) { + console.warn( + '[GestureLogger] GestureRecognition is unavailable. ' + + 'Make sure options.enableGestures() is called before xb.init().' + ); + return; + } + this._onGestureStart = (event) => { + const {hand, name, confidence = 0} = event.detail; + console.log( + `[gesture] ${hand} hand started ${name} (${confidence.toFixed(2)})` + ); + }; + this._onGestureEnd = (event) => { + const {hand, name} = event.detail; + console.log(`[gesture] ${hand} hand ended ${name}`); + }; + gestures.addEventListener('gesturestart', this._onGestureStart); + gestures.addEventListener('gestureend', this._onGestureEnd); + } + + dispose() { + const gestures = xb.core.gestureRecognition; + if (!gestures) return; + if (this._onGestureStart) { + gestures.removeEventListener('gesturestart', this._onGestureStart); + } + if (this._onGestureEnd) { + gestures.removeEventListener('gestureend', this._onGestureEnd); + } + } +} + +class GestureHUD extends xb.Script { + init() { + this._container = createHudElement(); + this._active = { + left: new Map(), + right: new Map(), + }; + this._labels = { + left: this._container.querySelector('[data-hand="left"]'), + right: this._container.querySelector('[data-hand="right"]'), + }; + + const gestures = xb.core.gestureRecognition; + if (!gestures) { + console.warn( + '[GestureHUD] GestureRecognition is unavailable. ' + + 'Make sure options.enableGestures() is called before xb.init().' + ); + return; + } + + const update = (event) => { + const {name, hand, confidence = 0} = event.detail; + this._active[hand].set(name, confidence); + this._refresh(hand); + }; + const clear = (event) => { + const {name, hand} = event.detail; + this._active[hand].delete(name); + this._refresh(hand); + }; + + this._onGestureStart = update; + this._onGestureUpdate = update; + this._onGestureEnd = clear; + + gestures.addEventListener('gesturestart', this._onGestureStart); + gestures.addEventListener('gestureupdate', this._onGestureUpdate); + gestures.addEventListener('gestureend', this._onGestureEnd); + } + + _refresh(hand) { + const label = this._labels[hand]; + if (!label) return; + const entries = this._active[hand]; + if (!entries || entries.size === 0) { + label.dataset.active = 'false'; + label.textContent = 'None'; + return; + } + let topGesture = 'None'; + let topConfidence = 0; + for (const [name, confidence] of entries.entries()) { + if (confidence >= topConfidence) { + topGesture = name; + topConfidence = confidence; + } + } + label.dataset.active = 'true'; + label.textContent = `${topGesture} (${topConfidence.toFixed(2)})`; + } + + dispose() { + const gestures = xb.core.gestureRecognition; + if (gestures) { + if (this._onGestureStart) { + gestures.removeEventListener('gesturestart', this._onGestureStart); + } + if (this._onGestureUpdate) { + gestures.removeEventListener('gestureupdate', this._onGestureUpdate); + } + if (this._onGestureEnd) { + gestures.removeEventListener('gestureend', this._onGestureEnd); + } + } + if (this._container?.parentElement) { + this._container.parentElement.removeChild(this._container); + } + } +} + +function start() { + xb.add(new GestureLogger()); + xb.add(new GestureHUD()); + xb.init(options); +} + +document.addEventListener('DOMContentLoaded', () => { + start(); +}); diff --git a/src/input/gestures/providers/HeuristicGestureDetectors.ts b/src/input/gestures/providers/HeuristicGestureDetectors.ts index 9ae6814..495c67b 100644 --- a/src/input/gestures/providers/HeuristicGestureDetectors.ts +++ b/src/input/gestures/providers/HeuristicGestureDetectors.ts @@ -28,6 +28,15 @@ function computePinch(context: HandContext, config: GestureConfiguration) { const index = getJoint(context, 'index-finger-tip'); if (!thumb || !index) return undefined; + const supportMetrics = (['middle', 'ring', 'pinky'] as FingerName[]) + .map((finger) => computeFingerMetric(context, finger)) + .filter(Boolean) as FingerMetrics[]; + const supportCurl = + supportMetrics.length > 0 + ? average(supportMetrics.map((metrics) => metrics.curlRatio)) + : 1; + const supportPenalty = clamp01((supportCurl - 1.05) / 0.35); + const handScale = estimateHandScale(context); const threshold = config.threshold ?? Math.max(0.018, handScale * 0.35); const distance = thumb.distanceTo(index); @@ -38,13 +47,15 @@ function computePinch(context: HandContext, config: GestureConfiguration) { const tightness = clamp01(1 - distance / (threshold * 0.85)); const loosePenalty = clamp01(1 - distance / (threshold * 1.4)); - const confidence = clamp01( + let confidence = clamp01( distance <= threshold ? tightness : loosePenalty * 0.4 ); + confidence *= 1 - supportPenalty * 0.45; + confidence = clamp01(confidence); return { confidence, - data: {distance, threshold}, + data: {distance, threshold, supportPenalty}, }; } @@ -53,6 +64,7 @@ function computeOpenPalm(context: HandContext, config: GestureConfiguration) { if (!fingerMetrics.length) return undefined; const handScale = estimateHandScale(context); const palmWidth = getPalmWidth(context) ?? handScale * 0.85; + const palmUp = getPalmUp(context); const extensionScores = fingerMetrics.map(({tipDistance}) => clamp01((tipDistance - handScale * 0.5) / (handScale * 0.45)) @@ -60,6 +72,14 @@ function computeOpenPalm(context: HandContext, config: GestureConfiguration) { const straightnessScores = fingerMetrics.map(({curlRatio}) => clamp01((curlRatio - 1.1) / 0.5) ); + const orientationScore = + palmUp && fingerMetrics.length + ? average( + fingerMetrics.map((metrics) => + fingerAlignmentScore(context, metrics, palmUp) + ) + ) + : 0.5; const neighbors = getAdjacentFingerDistances(context); const spreadScore = @@ -70,7 +90,10 @@ function computeOpenPalm(context: HandContext, config: GestureConfiguration) { const extensionScore = average(extensionScores); const straightScore = average(straightnessScores); const confidence = clamp01( - extensionScore * 0.5 + straightScore * 0.3 + spreadScore * 0.2 + extensionScore * 0.4 + + straightScore * 0.25 + + spreadScore * 0.2 + + orientationScore * 0.15 ); return { @@ -79,6 +102,7 @@ function computeOpenPalm(context: HandContext, config: GestureConfiguration) { extensionScore, straightScore, spreadScore, + orientationScore, threshold: config.threshold, }, }; @@ -102,13 +126,27 @@ function computeFist(context: HandContext, config: GestureConfiguration) { neighbors.average !== Infinity && palmWidth > EPSILON ? clamp01((palmWidth * 0.5 - neighbors.average) / (palmWidth * 0.35)) : 0; + const thumbTip = getJoint(context, 'thumb-tip'); + const indexBase = + getFingerJoint(context, 'index', 'phalanx-proximal') ?? + getFingerJoint(context, 'index', 'metacarpal'); + const thumbWrapScore = + thumbTip && indexBase && palmWidth > EPSILON + ? clamp01( + (palmWidth * 0.55 - thumbTip.distanceTo(indexBase)) / + (palmWidth * 0.35) + ) + : 0; const tipScore = clamp01( (handScale * 0.55 - tipAverage) / (handScale * 0.25) ); const curlScore = clamp01((1.08 - curlAverage) / 0.25); const confidence = clamp01( - tipScore * 0.5 + curlScore * 0.35 + clusterScore * 0.15 + tipScore * 0.45 + + curlScore * 0.3 + + clusterScore * 0.1 + + thumbWrapScore * 0.15 ); return { @@ -117,6 +155,7 @@ function computeFist(context: HandContext, config: GestureConfiguration) { tipAverage, curlAverage, clusterScore, + thumbWrapScore, threshold: config.threshold, }, }; @@ -162,8 +201,8 @@ function computeThumbsUp(context: HandContext, config: GestureConfiguration) { } const confidence = clamp01( - thumbExtendedScore * 0.35 + - curledScore * 0.3 + + thumbExtendedScore * 0.3 + + curledScore * 0.35 + orientationScore * 0.2 + separationScore * 0.15 ); @@ -189,16 +228,34 @@ function computePoint(context: HandContext, config: GestureConfiguration) { if (!otherMetrics.length) return undefined; const handScale = estimateHandScale(context); + const palmWidth = getPalmWidth(context) ?? handScale * 0.85; + const palmUp = getPalmUp(context); const indexCurlScore = clamp01((indexMetrics.curlRatio - 1.2) / 0.35); const indexReachScore = clamp01( (indexMetrics.tipDistance - handScale * 0.6) / (handScale * 0.25) ); + const indexDirectionScore = + palmUp && indexMetrics + ? fingerAlignmentScore(context, indexMetrics, palmUp) + : 0.4; const othersCurl = average(otherMetrics.map((metrics) => metrics.curlRatio)); const othersCurledScore = clamp01((1.05 - othersCurl) / 0.25); + const thumbTip = getJoint(context, 'thumb-tip'); + const thumbTuckedScore = + thumbTip && indexMetrics.metacarpal && palmWidth > EPSILON + ? clamp01( + (palmWidth * 0.75 - thumbTip.distanceTo(indexMetrics.metacarpal)) / + (palmWidth * 0.4) + ) + : 0.5; const confidence = clamp01( - indexCurlScore * 0.45 + indexReachScore * 0.25 + othersCurledScore * 0.3 + indexCurlScore * 0.35 + + indexReachScore * 0.25 + + othersCurledScore * 0.2 + + indexDirectionScore * 0.1 + + thumbTuckedScore * 0.1 ); return { @@ -207,6 +264,8 @@ function computePoint(context: HandContext, config: GestureConfiguration) { indexCurlScore, indexReachScore, othersCurledScore, + indexDirectionScore, + thumbTuckedScore, threshold: config.threshold, }, }; @@ -219,6 +278,7 @@ function computeSpread(context: HandContext, config: GestureConfiguration) { const handScale = estimateHandScale(context); const palmWidth = getPalmWidth(context) ?? handScale * 0.85; const neighbors = getAdjacentFingerDistances(context); + const palmUp = getPalmUp(context); const spreadScore = neighbors.average !== Infinity && palmWidth > EPSILON @@ -227,14 +287,25 @@ function computeSpread(context: HandContext, config: GestureConfiguration) { const extensionScore = clamp01( (average(fingerMetrics.map((metrics) => metrics.curlRatio)) - 1.15) / 0.45 ); + const orientationScore = + palmUp && fingerMetrics.length + ? average( + fingerMetrics.map((metrics) => + fingerAlignmentScore(context, metrics, palmUp) + ) + ) + : 0.5; - const confidence = clamp01(spreadScore * 0.6 + extensionScore * 0.4); + const confidence = clamp01( + spreadScore * 0.55 + extensionScore * 0.3 + orientationScore * 0.15 + ); return { confidence, data: { spreadScore, extensionScore, + orientationScore, threshold: config.threshold, }, }; @@ -392,6 +463,19 @@ function getFingerJoint( return getJoint(context, `${prefix}-${suffix}`); } +function fingerAlignmentScore( + context: HandContext, + metrics: FingerMetrics, + palmUp: THREE.Vector3 +) { + const base = metrics.metacarpal ?? getJoint(context, 'wrist'); + if (!base) return 0; + const direction = new THREE.Vector3().subVectors(metrics.tip, base); + if (direction.lengthSq() === 0) return 0; + direction.normalize(); + return clamp01((direction.dot(palmUp) - 0.35) / 0.5); +} + function clamp01(value: number) { return THREE.MathUtils.clamp(value, 0, 1); } diff --git a/templates/heuristic_hand_gestures/README.md b/templates/heuristic_hand_gestures/README.md index 40e51d1..d0ae743 100644 --- a/templates/heuristic_hand_gestures/README.md +++ b/templates/heuristic_hand_gestures/README.md @@ -6,3 +6,5 @@ open-palm, fist, thumbs-up, point, and spread detections on devices like Quest or in the simulator. For gesture configuration details, see https://xrblocks.github.io/docs/manual/HandGestures/. + +For an on-screen HUD and richer feedback, see `samples/gestures_heuristic/`.