From 49984c948e81dc54b719bbec4f0e54a784389301 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 14 Aug 2026 11:56:30 +0530 Subject: [PATCH 1/6] feat(voice): animate the mascot's mouth during realtime voice calls The mascot sat with a frozen mouth for the whole realtime call while tap-and-speak animated it, so the path we are moving users onto was the one that looked broken. The existing lip-sync cannot carry over as-is. It samples a viseme timeline against our own audio element's clock, and the realtime SDK owns playback - there is no currentMs() to sample and no timeline to sample against. What the SDK does expose is the output signal, so the mouth is driven from loudness instead: less accurate, since it moves with the envelope rather than forming phonemes, but in sync by construction because it is the audio being played rather than a prediction of it. The signal is read per animation frame out of a ref, not React state - routing 60fps through state would reconcile the page every frame, which is the cost the chat panel is memoised to avoid (#5357). State commits only when the visible mouth shape changes, which is at most four values. Rests the mouth on every exit rather than freezing on the last shape: turn ends, session ends mid-speech, unmount, a non-finite reading, or the SDK analyser throwing mid-frame (uncaught, that would kill the loop and leave the mouth open for the rest of the call). The classic path keeps ownership whenever the agent is not speaking, so the two sources never drive the same frame. Stage one of two. The alignment-driven viseme version is the follow-up; this stays as its fallback for when alignment is absent or the timeline runs dry. --- app/src/features/human/HumanPage.tsx | 32 +++++-- .../human/RealtimeVoiceControls.test.tsx | 1 + .../features/human/RealtimeVoiceControls.tsx | 36 +++++++- .../human/voice/amplitudeLipsync.test.ts | 46 ++++++++++ .../features/human/voice/amplitudeLipsync.ts | 83 ++++++++++++++++++ .../human/voice/useAmplitudeLipsync.test.ts | 84 +++++++++++++++++++ .../human/voice/useAmplitudeLipsync.ts | 79 +++++++++++++++++ .../human/voice/useRealtimeVoiceSession.ts | 7 ++ 8 files changed, 357 insertions(+), 11 deletions(-) create mode 100644 app/src/features/human/voice/amplitudeLipsync.test.ts create mode 100644 app/src/features/human/voice/amplitudeLipsync.ts create mode 100644 app/src/features/human/voice/useAmplitudeLipsync.test.ts create mode 100644 app/src/features/human/voice/useAmplitudeLipsync.ts diff --git a/app/src/features/human/HumanPage.tsx b/app/src/features/human/HumanPage.tsx index 08ea7642aa..251597642a 100644 --- a/app/src/features/human/HumanPage.tsx +++ b/app/src/features/human/HumanPage.tsx @@ -1,4 +1,4 @@ -import { useMemo } from 'react'; +import { useMemo, useRef } from 'react'; import { useT } from '../../lib/i18n/I18nContext'; import { useAppDispatch, useAppSelector } from '../../store/hooks'; @@ -22,6 +22,8 @@ import { import { useMascotManifest } from './Mascot/manifest/useMascotManifest'; import RealtimeVoiceControls from './RealtimeVoiceControls'; import { useHumanMascot } from './useHumanMascot'; +import { IDLE_REALTIME_VOICE_AUDIO, type RealtimeVoiceAudio } from './voice/amplitudeLipsync'; +import { useAmplitudeLipsync } from './voice/useAmplitudeLipsync'; import { resolveHumanVoiceEntry } from './voiceEntry'; const HumanPage = () => { @@ -35,6 +37,18 @@ const HumanPage = () => { const speakReplies = useAppSelector(selectSpeakReplies); const { face, visemeCode } = useHumanMascot({ speakReplies }); + + // Lip-sync for the realtime voice session. The session lives inside + // RealtimeVoiceControls (which owns its own ConversationProvider), so it + // publishes its output-loudness accessor into this ref and the mascot samples + // it per frame — a 60fps signal must not travel through React state. + const realtimeAudioRef = useRef({ ...IDLE_REALTIME_VOICE_AUDIO }); + const realtimeLipsync = useAmplitudeLipsync(realtimeAudioRef); + + // While the agent is speaking its own audio drives the mouth; otherwise the + // classic path keeps ownership, so the two never fight over the same frame. + const mascotFace = realtimeLipsync.active ? 'speaking' : face; + const mascotVisemeCode = realtimeLipsync.active ? realtimeLipsync.visemeCode : visemeCode; const mascotColor = useAppSelector(selectMascotColor); const customPrimary = useAppSelector(selectCustomPrimaryColor); const customSecondary = useAppSelector(selectCustomSecondaryColor); @@ -76,7 +90,9 @@ const HumanPage = () => { : null} + voiceChatControl={ + voiceEntry === 'realtime' ? : null + } showMicComposer={voiceEntry !== 'realtime'} projectThreadList /> @@ -97,23 +113,23 @@ const HumanPage = () => {
{customMascotGifUrl ? ( - + ) : mascotEntry ? ( ) : ( )} @@ -125,7 +141,7 @@ const HumanPage = () => { tap-and-speak rather than a second button stacked on it. */} {voiceEntry === 'both' && (
- +
)} diff --git a/app/src/features/human/RealtimeVoiceControls.test.tsx b/app/src/features/human/RealtimeVoiceControls.test.tsx index 18301a8079..796318d09f 100644 --- a/app/src/features/human/RealtimeVoiceControls.test.tsx +++ b/app/src/features/human/RealtimeVoiceControls.test.tsx @@ -31,6 +31,7 @@ function makeSession(overrides: Partial = {}): RealtimeVoi state: 'idle', isSpeaking: false, mode: 'listening', + getOutputVolume: () => 0, error: null, start, stop, diff --git a/app/src/features/human/RealtimeVoiceControls.tsx b/app/src/features/human/RealtimeVoiceControls.tsx index 01836fefbd..73a8b2f4f1 100644 --- a/app/src/features/human/RealtimeVoiceControls.tsx +++ b/app/src/features/human/RealtimeVoiceControls.tsx @@ -1,9 +1,11 @@ import { ConversationProvider } from '@elevenlabs/react'; +import { type RefObject, useEffect } from 'react'; import Button from '../../components/ui/Button'; import { useT } from '../../lib/i18n/I18nContext'; import { useAppSelector } from '../../store/hooks'; import { selectEffectiveMascotVoiceId } from '../../store/mascotSlice'; +import type { RealtimeVoiceAudio } from './voice/amplitudeLipsync'; import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession'; /** @@ -12,7 +14,7 @@ import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession'; * Wraps its own `ConversationProvider` (required by `@elevenlabs/react`) so it * stays self-contained and adds no context to the rest of the app. */ -function RealtimeVoiceControlsInner() { +function RealtimeVoiceControlsInner({ audioRef }: { audioRef?: RefObject }) { const { t } = useT(); const voiceId = useAppSelector(selectEffectiveMascotVoiceId); const session = useRealtimeVoiceSession({ voiceId }); @@ -20,6 +22,29 @@ function RealtimeVoiceControlsInner() { const active = session.state === 'active'; const connecting = session.state === 'connecting'; + // Publish the output-loudness accessor for the mascot's lip-sync. Written into + // a ref rather than lifted into state because the mascot samples it once per + // animation frame; see `useAmplitudeLipsync`. The session lives under this + // component's own ConversationProvider, so this is the only place that can + // reach it. + const { getOutputVolume, isSpeaking } = session; + useEffect(() => { + if (!audioRef) return; + audioRef.current.getOutputVolume = active ? getOutputVolume : null; + audioRef.current.speaking = active && isSpeaking; + }, [audioRef, active, isSpeaking, getOutputVolume]); + + // A session that ends mid-speech would otherwise leave `speaking` true and the + // mouth frozen open. + useEffect( + () => () => { + if (!audioRef) return; + audioRef.current.getOutputVolume = null; + audioRef.current.speaking = false; + }, + [audioRef] + ); + const label = connecting ? t('voice.mode.connecting') : active @@ -51,10 +76,15 @@ function RealtimeVoiceControlsInner() { ); } -export default function RealtimeVoiceControls() { +export default function RealtimeVoiceControls({ + audioRef, +}: { + /** Optional sink for the mascot's lip-sync signal (see RealtimeVoiceAudio). */ + audioRef?: RefObject; +}) { return ( - + ); } diff --git a/app/src/features/human/voice/amplitudeLipsync.test.ts b/app/src/features/human/voice/amplitudeLipsync.test.ts new file mode 100644 index 0000000000..4f8d88a3c6 --- /dev/null +++ b/app/src/features/human/voice/amplitudeLipsync.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; + +import { amplitudeToVisemeCode, smoothAmplitude } from './amplitudeLipsync'; + +describe('amplitudeToVisemeCode', () => { + it('rests the mouth on silence and room tone', () => { + expect(amplitudeToVisemeCode(0)).toBe('sil'); + // Below the floor: the tail of a word, not speech. Holding the mouth open + // through these gaps is what makes naive amplitude lip-sync look slack. + expect(amplitudeToVisemeCode(0.03)).toBe('sil'); + }); + + it('opens the mouth further as the signal gets louder', () => { + const codes = [0.08, 0.2, 0.6].map(amplitudeToVisemeCode); + expect(codes).toEqual(['I', 'E', 'aa']); + }); + + // A garbage reading (analyser torn down mid-frame) must not map to a wide-open + // mouth that then sticks for the rest of the call. Infinity rests for the same + // reason NaN does: it is a broken sample, not a loud one. + it('rests on a non-finite reading', () => { + expect(amplitudeToVisemeCode(Number.NaN)).toBe('sil'); + expect(amplitudeToVisemeCode(Number.POSITIVE_INFINITY)).toBe('sil'); + }); +}); + +describe('smoothAmplitude', () => { + it('moves toward the new sample', () => { + expect(smoothAmplitude(0, 1)).toBeGreaterThan(0); + expect(smoothAmplitude(1, 0)).toBeLessThan(1); + }); + + // Asymmetric on purpose: consonant onsets must land on time, but the mouth + // must not snap shut inside a word. + it('opens faster than it closes', () => { + const opening = smoothAmplitude(0, 1) - 0; + const closing = 1 - smoothAmplitude(1, 0); + expect(opening).toBeGreaterThan(closing); + }); + + it('converges on a held level rather than oscillating', () => { + let level = 0; + for (let i = 0; i < 40; i += 1) level = smoothAmplitude(level, 0.5); + expect(level).toBeCloseTo(0.5, 2); + }); +}); diff --git a/app/src/features/human/voice/amplitudeLipsync.ts b/app/src/features/human/voice/amplitudeLipsync.ts new file mode 100644 index 0000000000..0515a8e2b5 --- /dev/null +++ b/app/src/features/human/voice/amplitudeLipsync.ts @@ -0,0 +1,83 @@ +/** + * Amplitude-driven lip-sync for the realtime voice session (#5399). + * + * The classic tap-and-speak path animates the mouth from a viseme *timeline*: + * frames of `{viseme, ms}` sampled against our own audio element's clock. That + * is not available here — the realtime SDK owns playback, so there is no + * `currentMs()` to sample and no per-character timing unless we subscribe to + * alignment events and reconstruct one. + * + * What the SDK does expose is the output signal itself (`getOutputVolume()`), + * so the mouth is driven from loudness instead. That is genuinely less accurate + * — it opens and closes with the envelope rather than forming phonemes, so no + * `M`/`F` closures — but it is in sync *by construction*, because it is the + * audio being played rather than a prediction of it. A frozen mouth while the + * agent talks reads as broken; an approximate one reads as alive. + * + * The viseme-timeline version is the follow-up, and this is the fallback it + * would keep for the case where alignment is absent or has run dry. + */ + +/** Viseme codes, ordered by how open the mouth is. */ +const REST = 'sil'; +const NARROW = 'I'; // openness 0.30 +const MID = 'E'; // openness 0.45 +const OPEN = 'aa'; // openness ~0.95 + +/** + * Below this the signal is room tone or the tail of a word, not speech. Holding + * the mouth open through those gaps is what makes naive amplitude lip-sync look + * slack-jawed, so anything under it rests. + */ +const SILENCE_FLOOR = 0.04; + +/** Where the mouth steps from narrow to mid, and from mid to wide open. */ +const MID_THRESHOLD = 0.12; +const OPEN_THRESHOLD = 0.28; + +/** + * Smoothing applied to the raw reading, as the weight given to the new sample. + * + * `getOutputVolume()` is sampled per animation frame and is noisy at that rate: + * fed straight through it produces a chattering mouth that reads as a glitch + * rather than as speech. Asymmetric on purpose — opening tracks the signal + * quickly so consonant onsets land on time, closing lags so the mouth does not + * snap shut inside a word. + */ +const ATTACK = 0.55; +const RELEASE = 0.18; + +/** Smooth one amplitude sample toward the previous level. Pure + unit-tested. */ +export function smoothAmplitude(previous: number, sample: number): number { + const weight = sample > previous ? ATTACK : RELEASE; + return previous + (sample - previous) * weight; +} + +/** + * Map a smoothed amplitude (0..1) onto a viseme code. Steps rather than + * interpolates because the Rive mouth is driven by a code, not a scalar. + * Pure + unit-tested. + */ +export function amplitudeToVisemeCode(level: number): string { + if (!Number.isFinite(level) || level < SILENCE_FLOOR) return REST; + if (level < MID_THRESHOLD) return NARROW; + if (level < OPEN_THRESHOLD) return MID; + return OPEN; +} + +/** + * What the realtime controls publish for the mascot to read. Held in a ref and + * mutated in place: the mascot samples it once per animation frame, and routing + * a 60fps signal through React state would re-render the page on every frame. + */ +export interface RealtimeVoiceAudio { + /** SDK accessor for output loudness, or null when no session is live. */ + getOutputVolume: (() => number) | null; + /** Whether the agent is currently speaking (SDK `isSpeaking`). */ + speaking: boolean; +} + +export const IDLE_REALTIME_VOICE_AUDIO: RealtimeVoiceAudio = { + getOutputVolume: null, + speaking: false, +}; diff --git a/app/src/features/human/voice/useAmplitudeLipsync.test.ts b/app/src/features/human/voice/useAmplitudeLipsync.test.ts new file mode 100644 index 0000000000..4d9d9e9f1e --- /dev/null +++ b/app/src/features/human/voice/useAmplitudeLipsync.test.ts @@ -0,0 +1,84 @@ +import { act, renderHook } from '@testing-library/react'; +import { createRef } from 'react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { type RealtimeVoiceAudio } from './amplitudeLipsync'; +import { useAmplitudeLipsync } from './useAmplitudeLipsync'; + +/** Drive the rAF loop by hand so frames are deterministic. */ +let frames: FrameRequestCallback[] = []; + +function flushFrames(count: number): void { + for (let i = 0; i < count; i += 1) { + const pending = frames; + frames = []; + act(() => pending.forEach(cb => cb(performance.now()))); + } +} + +function audioRef(overrides: Partial = {}) { + const ref = createRef() as { current: RealtimeVoiceAudio }; + ref.current = { getOutputVolume: null, speaking: false, ...overrides }; + return ref; +} + +describe('useAmplitudeLipsync', () => { + beforeEach(() => { + frames = []; + vi.stubGlobal('requestAnimationFrame', (cb: FrameRequestCallback) => { + frames.push(cb); + return frames.length; + }); + vi.stubGlobal('cancelAnimationFrame', () => {}); + }); + afterEach(() => vi.unstubAllGlobals()); + + it('stays inactive and rested while nothing is speaking', () => { + const { result } = renderHook(() => useAmplitudeLipsync(audioRef())); + flushFrames(3); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + }); + + it('drives the mouth from output loudness while the agent speaks', () => { + const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); + const { result } = renderHook(() => useAmplitudeLipsync(ref)); + flushFrames(10); + expect(result.current.active).toBe(true); + expect(result.current.visemeCode).toBe('aa'); + }); + + // The mouth must return to rest when the turn ends, not freeze on its last + // shape — a stuck-open mascot is worse than one that never moved. + it('rests the mouth when speaking stops', () => { + const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); + const { result } = renderHook(() => useAmplitudeLipsync(ref)); + flushFrames(10); + expect(result.current.visemeCode).toBe('aa'); + + ref.current.speaking = false; + flushFrames(2); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + }); + + // The SDK reads a live analyser; a session torn down mid-frame throws rather + // than returning 0. An uncaught throw would kill the loop and freeze the mouth. + it('survives an accessor that throws mid-session', () => { + const ref = audioRef({ + speaking: true, + getOutputVolume: () => { + throw new Error('analyser closed'); + }, + }); + const { result } = renderHook(() => useAmplitudeLipsync(ref)); + expect(() => flushFrames(5)).not.toThrow(); + expect(result.current.visemeCode).toBe('sil'); + }); + + it('cancels its frame loop on unmount', () => { + const cancel = vi.fn(); + vi.stubGlobal('cancelAnimationFrame', cancel); + const { unmount } = renderHook(() => useAmplitudeLipsync(audioRef())); + unmount(); + expect(cancel).toHaveBeenCalled(); + }); +}); diff --git a/app/src/features/human/voice/useAmplitudeLipsync.ts b/app/src/features/human/voice/useAmplitudeLipsync.ts new file mode 100644 index 0000000000..4e8b798fef --- /dev/null +++ b/app/src/features/human/voice/useAmplitudeLipsync.ts @@ -0,0 +1,79 @@ +import { type RefObject, useEffect, useRef, useState } from 'react'; + +import { + amplitudeToVisemeCode, + type RealtimeVoiceAudio, + smoothAmplitude, +} from './amplitudeLipsync'; + +export interface AmplitudeLipsync { + /** True while the realtime agent is speaking and driving the mouth. */ + active: boolean; + /** Viseme code for Rive's `mouthVisemeCode` input. */ + visemeCode: string; +} + +/** + * Drive the mascot's mouth from the realtime session's output loudness. + * + * Runs an animation-frame loop only while the agent is speaking, so an idle + * Human tab schedules no frames. Reads the SDK accessor out of a ref rather + * than props because the session lives inside `RealtimeVoiceControls` (which + * owns its own `ConversationProvider`) — see `RealtimeVoiceAudio`. + * + * State is committed only when the viseme code actually changes. The smoothed + * level moves every frame, but the code it maps to steps between four values, + * so re-rendering on the raw level would reconcile the page ~60 times a second + * to produce the same mouth — the exact cost the chat panel is memoised to + * avoid (#5357). + */ +export function useAmplitudeLipsync(audio: RefObject): AmplitudeLipsync { + const [state, setState] = useState({ active: false, visemeCode: 'sil' }); + const levelRef = useRef(0); + // Mirrors `state` for the loop to compare against without re-subscribing the + // effect on every change. + const codeRef = useRef('sil'); + const activeRef = useRef(false); + + useEffect(() => { + let raf = 0; + let stopped = false; + + const commit = (active: boolean, visemeCode: string): void => { + if (active === activeRef.current && visemeCode === codeRef.current) return; + activeRef.current = active; + codeRef.current = visemeCode; + setState({ active, visemeCode }); + }; + + const tick = (): void => { + if (stopped) return; + const { getOutputVolume, speaking } = audio.current; + if (!speaking || !getOutputVolume) { + levelRef.current = 0; + commit(false, 'sil'); + } else { + // The SDK reads from a live analyser; a session torn down mid-frame + // makes this throw rather than return 0, which would kill the loop and + // freeze the mouth open for the rest of the call. + let sample = 0; + try { + sample = getOutputVolume(); + } catch { + sample = 0; + } + levelRef.current = smoothAmplitude(levelRef.current, sample); + commit(true, amplitudeToVisemeCode(levelRef.current)); + } + raf = window.requestAnimationFrame(tick); + }; + + raf = window.requestAnimationFrame(tick); + return () => { + stopped = true; + window.cancelAnimationFrame(raf); + }; + }, [audio]); + + return state; +} diff --git a/app/src/features/human/voice/useRealtimeVoiceSession.ts b/app/src/features/human/voice/useRealtimeVoiceSession.ts index f6d355a721..87bd5614a5 100644 --- a/app/src/features/human/voice/useRealtimeVoiceSession.ts +++ b/app/src/features/human/voice/useRealtimeVoiceSession.ts @@ -31,6 +31,12 @@ export interface RealtimeVoiceSession { isSpeaking: boolean; /** ElevenLabs turn mode; `listening` while the user speaks. */ mode: 'speaking' | 'listening'; + /** + * Output loudness (0..1) of the agent's voice, sampled from the SDK's + * analyser. Drives the mascot's amplitude lip-sync — the realtime SDK owns + * playback, so this is the only signal available to animate the mouth against. + */ + getOutputVolume: () => number; error: string | null; /** Fetch a signed URL and open the WebSocket session. Idempotent while busy. */ start: () => Promise; @@ -174,6 +180,7 @@ export function useRealtimeVoiceSession(opts?: { voiceId?: string }): RealtimeVo state, isSpeaking: conversation.isSpeaking, mode: conversation.mode, + getOutputVolume: conversation.getOutputVolume, error, start, stop, From 02b4d677378cb3b547499a28526350cfc42dc9eb Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 14 Aug 2026 12:12:10 +0530 Subject: [PATCH 2/6] fix(voice): reset the lip-sync level on an invalid analyser reading Review caught two ways a bad sample outlives itself, and the second is worse than it looks. Smoothing toward 0 on a throw decays instead of clearing: from a loud sample it takes ~16 frames to fall under the silence floor, so the mouth stays open a quarter of a second after the audio is already gone. Worse, a non-finite sample poisons the level permanently. smoothAmplitude carries NaN through every later frame, so no valid sample can ever recover it - the mouth never animates again for the rest of the call. Guarding only at the viseme mapping hid that as a quiet mouth rather than an error. Both now reset the level and rest on the next frame. The existing test passed against the broken code because it started from silence and never exercised a loud-to-invalid transition; the two added tests fail without this change. --- .../human/voice/useAmplitudeLipsync.test.ts | 40 +++++++++++++++++++ .../human/voice/useAmplitudeLipsync.ts | 24 +++++++---- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/app/src/features/human/voice/useAmplitudeLipsync.test.ts b/app/src/features/human/voice/useAmplitudeLipsync.test.ts index 4d9d9e9f1e..6207ad15c1 100644 --- a/app/src/features/human/voice/useAmplitudeLipsync.test.ts +++ b/app/src/features/human/voice/useAmplitudeLipsync.test.ts @@ -74,6 +74,46 @@ describe('useAmplitudeLipsync', () => { expect(result.current.visemeCode).toBe('sil'); }); + // Smoothing toward 0 would hold the mouth open for ~16 frames after the audio + // is already gone, so a bad reading resets on the very next frame. + it('rests immediately when the accessor throws after a loud sample', () => { + let volume = 0.9; + const ref = audioRef({ + speaking: true, + getOutputVolume: () => { + if (volume < 0) throw new Error('analyser closed'); + return volume; + }, + }); + const { result } = renderHook(() => useAmplitudeLipsync(ref)); + flushFrames(10); + expect(result.current.visemeCode).toBe('aa'); + + volume = -1; // next call throws + flushFrames(1); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + }); + + // A non-finite sample smoothed into the level poisons it permanently: every + // later sample stays NaN and the mouth never animates again for the rest of + // the call. Guarding only at the viseme mapping hides that as a quiet mouth. + it('recovers on the next valid sample after a non-finite reading', () => { + let volume = 0.9; + const ref = audioRef({ speaking: true, getOutputVolume: () => volume }); + const { result } = renderHook(() => useAmplitudeLipsync(ref)); + flushFrames(10); + expect(result.current.visemeCode).toBe('aa'); + + volume = Number.NaN; + flushFrames(1); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + + // The level must not have been poisoned — real audio animates again. + volume = 0.9; + flushFrames(10); + expect(result.current).toEqual({ active: true, visemeCode: 'aa' }); + }); + it('cancels its frame loop on unmount', () => { const cancel = vi.fn(); vi.stubGlobal('cancelAnimationFrame', cancel); diff --git a/app/src/features/human/voice/useAmplitudeLipsync.ts b/app/src/features/human/voice/useAmplitudeLipsync.ts index 4e8b798fef..9f746c03ed 100644 --- a/app/src/features/human/voice/useAmplitudeLipsync.ts +++ b/app/src/features/human/voice/useAmplitudeLipsync.ts @@ -53,17 +53,27 @@ export function useAmplitudeLipsync(audio: RefObject): Ampli levelRef.current = 0; commit(false, 'sil'); } else { - // The SDK reads from a live analyser; a session torn down mid-frame - // makes this throw rather than return 0, which would kill the loop and - // freeze the mouth open for the rest of the call. - let sample = 0; + // The SDK reads from a live analyser, so a session torn down mid-frame + // makes this throw rather than return 0. + let sample: number; try { sample = getOutputVolume(); } catch { - sample = 0; + sample = Number.NaN; + } + if (Number.isFinite(sample)) { + levelRef.current = smoothAmplitude(levelRef.current, sample); + commit(true, amplitudeToVisemeCode(levelRef.current)); + } else { + // A bad reading resets rather than decays, for two reasons. Smoothing + // toward 0 would hold the mouth open for ~16 frames after the audio is + // already gone; and smoothing toward a non-finite value poisons + // `levelRef` permanently — every later sample stays NaN, so the mouth + // never animates again for the rest of the call. Guarding only at the + // viseme mapping hides that as a quiet mouth instead of an error. + levelRef.current = 0; + commit(false, 'sil'); } - levelRef.current = smoothAmplitude(levelRef.current, sample); - commit(true, amplitudeToVisemeCode(levelRef.current)); } raf = window.requestAnimationFrame(tick); }; From eb0496eb551ca88674ece696e17a3e6177e267ef Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 14 Aug 2026 17:21:56 +0530 Subject: [PATCH 3/6] fix(voice): gate the lip-sync frame loop on the agent's speaking edge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit useAmplitudeLipsync scheduled a requestAnimationFrame loop from mount to unmount regardless of state: `tick` rescheduled unconditionally and the effect's only dependency was a ref, so it never re-subscribed. HumanPage mounts the hook unconditionally, so every Human-tab visitor — including the classic voice path where no realtime session exists and getOutputVolume is permanently null — paid a no-op callback ~60x/second, a wake/battery cost on the app's main screen. The hook's doc comment and the PR's Impact section both claimed an idle tab scheduled no frames; neither was true. Add an `enabled` parameter that gates the effect, and reset the mouth to rest when it goes false (the loop that would otherwise commit 'sil' no longer runs). RealtimeVoiceControls surfaces the agent's speaking edge (`active && isSpeaking`) via onSpeakingChange; HumanPage lifts it into state and passes it as `enabled`. The 60fps amplitude still travels through the ref — only the on/off edge, which flips a couple of times per turn, becomes state. Classic and idle tabs now schedule zero frames, making the documented invariant honest. Tests pin that a disabled hook schedules no frames and that disabling mid-turn tears the loop down (no further rAF) and rests the mouth, and cover the new onSpeakingChange wiring in RealtimeVoiceControls. Addresses review feedback from CodeGhost21 and tinysweeper on #5546. Co-Authored-By: Claude Opus 4.8 --- app/src/features/human/HumanPage.tsx | 22 +++++-- .../human/RealtimeVoiceControls.test.tsx | 27 ++++++++- .../features/human/RealtimeVoiceControls.tsx | 41 +++++++++---- .../human/voice/useAmplitudeLipsync.test.ts | 58 +++++++++++++++---- .../human/voice/useAmplitudeLipsync.ts | 37 +++++++++--- 5 files changed, 148 insertions(+), 37 deletions(-) diff --git a/app/src/features/human/HumanPage.tsx b/app/src/features/human/HumanPage.tsx index 251597642a..7df53840ce 100644 --- a/app/src/features/human/HumanPage.tsx +++ b/app/src/features/human/HumanPage.tsx @@ -1,4 +1,4 @@ -import { useMemo, useRef } from 'react'; +import { useMemo, useRef, useState } from 'react'; import { useT } from '../../lib/i18n/I18nContext'; import { useAppDispatch, useAppSelector } from '../../store/hooks'; @@ -43,7 +43,13 @@ const HumanPage = () => { // publishes its output-loudness accessor into this ref and the mascot samples // it per frame — a 60fps signal must not travel through React state. const realtimeAudioRef = useRef({ ...IDLE_REALTIME_VOICE_AUDIO }); - const realtimeLipsync = useAmplitudeLipsync(realtimeAudioRef); + // The agent's speaking edge, lifted out of RealtimeVoiceControls so it can gate + // the lip-sync loop below. Flips a couple of times per turn, so it is cheap as + // state (the 60fps amplitude stays in the ref). While it is false — an idle + // realtime session, or the classic voice path that never mounts the control — + // the loop schedules no frames at all. + const [realtimeSpeaking, setRealtimeSpeaking] = useState(false); + const realtimeLipsync = useAmplitudeLipsync(realtimeAudioRef, realtimeSpeaking); // While the agent is speaking its own audio drives the mouth; otherwise the // classic path keeps ownership, so the two never fight over the same frame. @@ -91,7 +97,12 @@ const HumanPage = () => { variant="sidebar" composer="mic-cloud" voiceChatControl={ - voiceEntry === 'realtime' ? : null + voiceEntry === 'realtime' ? ( + + ) : null } showMicComposer={voiceEntry !== 'realtime'} projectThreadList @@ -141,7 +152,10 @@ const HumanPage = () => { tap-and-speak rather than a second button stacked on it. */} {voiceEntry === 'both' && (
- +
)} diff --git a/app/src/features/human/RealtimeVoiceControls.test.tsx b/app/src/features/human/RealtimeVoiceControls.test.tsx index 796318d09f..42e49245de 100644 --- a/app/src/features/human/RealtimeVoiceControls.test.tsx +++ b/app/src/features/human/RealtimeVoiceControls.test.tsx @@ -49,11 +49,11 @@ const LABEL = { speaking: 'Speaking', } as const; -function renderControls() { +function renderControls(onSpeakingChange?: (speaking: boolean) => void) { const store = configureStore({ reducer: { mascot: mascotReducer } }); return render( - + ); } @@ -113,4 +113,27 @@ describe('RealtimeVoiceControls', () => { expect(stop).toHaveBeenCalledTimes(1); expect(start).not.toHaveBeenCalled(); }); + + // The speaking edge gates the mascot's lip-sync loop on the page + // (useAmplitudeLipsync's `enabled`), so it must reflect `active && isSpeaking`, + // not either half alone. + it('reports not-speaking to onSpeakingChange while idle', () => { + const onSpeakingChange = vi.fn(); + renderControls(onSpeakingChange); + expect(onSpeakingChange).toHaveBeenLastCalledWith(false); + }); + + it('reports speaking only when the session is active and the agent speaks', () => { + const onSpeakingChange = vi.fn(); + session = makeSession({ state: 'active', isSpeaking: true }); + renderControls(onSpeakingChange); + expect(onSpeakingChange).toHaveBeenLastCalledWith(true); + }); + + it('reports not-speaking when active but the agent is silent', () => { + const onSpeakingChange = vi.fn(); + session = makeSession({ state: 'active', isSpeaking: false }); + renderControls(onSpeakingChange); + expect(onSpeakingChange).toHaveBeenLastCalledWith(false); + }); }); diff --git a/app/src/features/human/RealtimeVoiceControls.tsx b/app/src/features/human/RealtimeVoiceControls.tsx index 73a8b2f4f1..ee26cd51a3 100644 --- a/app/src/features/human/RealtimeVoiceControls.tsx +++ b/app/src/features/human/RealtimeVoiceControls.tsx @@ -14,7 +14,13 @@ import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession'; * Wraps its own `ConversationProvider` (required by `@elevenlabs/react`) so it * stays self-contained and adds no context to the rest of the app. */ -function RealtimeVoiceControlsInner({ audioRef }: { audioRef?: RefObject }) { +function RealtimeVoiceControlsInner({ + audioRef, + onSpeakingChange, +}: { + audioRef?: RefObject; + onSpeakingChange?: (speaking: boolean) => void; +}) { const { t } = useT(); const voiceId = useAppSelector(selectEffectiveMascotVoiceId); const session = useRealtimeVoiceSession({ voiceId }); @@ -28,21 +34,31 @@ function RealtimeVoiceControlsInner({ audioRef }: { audioRef?: RefObject { - if (!audioRef) return; - audioRef.current.getOutputVolume = active ? getOutputVolume : null; - audioRef.current.speaking = active && isSpeaking; - }, [audioRef, active, isSpeaking, getOutputVolume]); + if (audioRef) { + audioRef.current.getOutputVolume = active ? getOutputVolume : null; + audioRef.current.speaking = speaking; + } + // Surface the speaking edge so the page can gate the mascot's lip-sync rAF + // loop (`useAmplitudeLipsync`'s `enabled`) — an idle or classic Human tab + // then schedules no frames. Unlike the 60fps amplitude above, this flips a + // couple of times per turn, so it is cheap to lift into React state. + onSpeakingChange?.(speaking); + }, [audioRef, active, speaking, getOutputVolume, onSpeakingChange]); // A session that ends mid-speech would otherwise leave `speaking` true and the - // mouth frozen open. + // mouth frozen open — reset both the ref the mascot samples and the speaking + // edge the page gates its loop on. useEffect( () => () => { - if (!audioRef) return; - audioRef.current.getOutputVolume = null; - audioRef.current.speaking = false; + if (audioRef) { + audioRef.current.getOutputVolume = null; + audioRef.current.speaking = false; + } + onSpeakingChange?.(false); }, - [audioRef] + [audioRef, onSpeakingChange] ); const label = connecting @@ -78,13 +94,16 @@ function RealtimeVoiceControlsInner({ audioRef }: { audioRef?: RefObject; + /** Notified with the agent's speaking edge so the page can gate the loop. */ + onSpeakingChange?: (speaking: boolean) => void; }) { return ( - + ); } diff --git a/app/src/features/human/voice/useAmplitudeLipsync.test.ts b/app/src/features/human/voice/useAmplitudeLipsync.test.ts index 6207ad15c1..0ebbdde83d 100644 --- a/app/src/features/human/voice/useAmplitudeLipsync.test.ts +++ b/app/src/features/human/voice/useAmplitudeLipsync.test.ts @@ -7,6 +7,8 @@ import { useAmplitudeLipsync } from './useAmplitudeLipsync'; /** Drive the rAF loop by hand so frames are deterministic. */ let frames: FrameRequestCallback[] = []; +let raf: ReturnType; +let cancel: ReturnType; function flushFrames(count: number): void { for (let i = 0; i < count; i += 1) { @@ -25,23 +27,25 @@ function audioRef(overrides: Partial = {}) { describe('useAmplitudeLipsync', () => { beforeEach(() => { frames = []; - vi.stubGlobal('requestAnimationFrame', (cb: FrameRequestCallback) => { + raf = vi.fn((cb: FrameRequestCallback) => { frames.push(cb); return frames.length; }); - vi.stubGlobal('cancelAnimationFrame', () => {}); + cancel = vi.fn(); + vi.stubGlobal('requestAnimationFrame', raf); + vi.stubGlobal('cancelAnimationFrame', cancel); }); afterEach(() => vi.unstubAllGlobals()); it('stays inactive and rested while nothing is speaking', () => { - const { result } = renderHook(() => useAmplitudeLipsync(audioRef())); + const { result } = renderHook(() => useAmplitudeLipsync(audioRef(), true)); flushFrames(3); expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); }); it('drives the mouth from output loudness while the agent speaks', () => { const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); - const { result } = renderHook(() => useAmplitudeLipsync(ref)); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); flushFrames(10); expect(result.current.active).toBe(true); expect(result.current.visemeCode).toBe('aa'); @@ -51,7 +55,7 @@ describe('useAmplitudeLipsync', () => { // shape — a stuck-open mascot is worse than one that never moved. it('rests the mouth when speaking stops', () => { const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); - const { result } = renderHook(() => useAmplitudeLipsync(ref)); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); flushFrames(10); expect(result.current.visemeCode).toBe('aa'); @@ -69,7 +73,7 @@ describe('useAmplitudeLipsync', () => { throw new Error('analyser closed'); }, }); - const { result } = renderHook(() => useAmplitudeLipsync(ref)); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); expect(() => flushFrames(5)).not.toThrow(); expect(result.current.visemeCode).toBe('sil'); }); @@ -85,7 +89,7 @@ describe('useAmplitudeLipsync', () => { return volume; }, }); - const { result } = renderHook(() => useAmplitudeLipsync(ref)); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); flushFrames(10); expect(result.current.visemeCode).toBe('aa'); @@ -100,7 +104,7 @@ describe('useAmplitudeLipsync', () => { it('recovers on the next valid sample after a non-finite reading', () => { let volume = 0.9; const ref = audioRef({ speaking: true, getOutputVolume: () => volume }); - const { result } = renderHook(() => useAmplitudeLipsync(ref)); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); flushFrames(10); expect(result.current.visemeCode).toBe('aa'); @@ -115,10 +119,42 @@ describe('useAmplitudeLipsync', () => { }); it('cancels its frame loop on unmount', () => { - const cancel = vi.fn(); - vi.stubGlobal('cancelAnimationFrame', cancel); - const { unmount } = renderHook(() => useAmplitudeLipsync(audioRef())); + const { unmount } = renderHook(() => useAmplitudeLipsync(audioRef(), true)); unmount(); expect(cancel).toHaveBeenCalled(); }); + + // The loop is the whole cost of the feature, and it must not run when the tab + // is not driving a speaking session — a disabled hook schedules no frames at + // all, even with a live-looking accessor sitting in the ref (#5546). + it('schedules no frames while disabled', () => { + const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); + const { result } = renderHook(() => useAmplitudeLipsync(ref, false)); + flushFrames(3); + expect(raf).not.toHaveBeenCalled(); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + }); + + // Disabling mid-turn must tear the loop down, not just idle it: no further + // frames are queued, and the mouth resets rather than freezing on its last + // shape (the loop that would otherwise commit 'sil' is gone). + it('stops scheduling frames and rests when it becomes disabled', () => { + const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); + const { result, rerender } = renderHook(({ enabled }) => useAmplitudeLipsync(ref, enabled), { + initialProps: { enabled: true }, + }); + flushFrames(10); + expect(result.current.visemeCode).toBe('aa'); + const scheduledWhileActive = raf.mock.calls.length; + expect(scheduledWhileActive).toBeGreaterThan(0); + + rerender({ enabled: false }); + expect(cancel).toHaveBeenCalled(); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + + // No new frames after the idle transition — pins the regression the gate + // exists to prevent (the loop used to reschedule unconditionally). + flushFrames(5); + expect(raf.mock.calls.length).toBe(scheduledWhileActive); + }); }); diff --git a/app/src/features/human/voice/useAmplitudeLipsync.ts b/app/src/features/human/voice/useAmplitudeLipsync.ts index 9f746c03ed..d8fec91ebd 100644 --- a/app/src/features/human/voice/useAmplitudeLipsync.ts +++ b/app/src/features/human/voice/useAmplitudeLipsync.ts @@ -16,10 +16,16 @@ export interface AmplitudeLipsync { /** * Drive the mascot's mouth from the realtime session's output loudness. * - * Runs an animation-frame loop only while the agent is speaking, so an idle - * Human tab schedules no frames. Reads the SDK accessor out of a ref rather - * than props because the session lives inside `RealtimeVoiceControls` (which - * owns its own `ConversationProvider`) — see `RealtimeVoiceAudio`. + * The animation-frame loop exists only while `enabled` is true — the caller + * passes the agent's speaking edge, so an idle Human tab (and the classic + * voice path, which never speaks a realtime session) schedules no frames at + * all. `enabled` gates the effect rather than being read inside the loop + * because a ref cannot re-subscribe an effect: the on/off edge changes a couple + * of times per turn, so it is cheap as a dependency, while the amplitude it + * drives stays in a ref because that moves every frame. Reads the SDK accessor + * out of a ref rather than props because the session lives inside + * `RealtimeVoiceControls` (which owns its own `ConversationProvider`) — see + * `RealtimeVoiceAudio`. * * State is committed only when the viseme code actually changes. The smoothed * level moves every frame, but the code it maps to steps between four values, @@ -27,7 +33,10 @@ export interface AmplitudeLipsync { * to produce the same mouth — the exact cost the chat panel is memoised to * avoid (#5357). */ -export function useAmplitudeLipsync(audio: RefObject): AmplitudeLipsync { +export function useAmplitudeLipsync( + audio: RefObject, + enabled: boolean +): AmplitudeLipsync { const [state, setState] = useState({ active: false, visemeCode: 'sil' }); const levelRef = useRef(0); // Mirrors `state` for the loop to compare against without re-subscribing the @@ -36,9 +45,6 @@ export function useAmplitudeLipsync(audio: RefObject): Ampli const activeRef = useRef(false); useEffect(() => { - let raf = 0; - let stopped = false; - const commit = (active: boolean, visemeCode: string): void => { if (active === activeRef.current && visemeCode === codeRef.current) return; activeRef.current = active; @@ -46,6 +52,19 @@ export function useAmplitudeLipsync(audio: RefObject): Ampli setState({ active, visemeCode }); }; + // Disabled: schedule nothing, and make sure the mouth is at rest. The loop + // is the only thing that commits 'sil' when speech stops, and it does not + // run here, so the reset has to happen on the disabling render instead — + // otherwise the mouth would freeze on its last shape. + if (!enabled) { + levelRef.current = 0; + commit(false, 'sil'); + return; + } + + let raf = 0; + let stopped = false; + const tick = (): void => { if (stopped) return; const { getOutputVolume, speaking } = audio.current; @@ -83,7 +102,7 @@ export function useAmplitudeLipsync(audio: RefObject): Ampli stopped = true; window.cancelAnimationFrame(raf); }; - }, [audio]); + }, [audio, enabled]); return state; } From 6974b9f338a29dd93539e7091d568da02c5d6481 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 14 Aug 2026 17:36:20 +0530 Subject: [PATCH 4/6] fix(voice): guard audioRef.current and cover the publication path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address tinysweeper review on #5546: - Guard `audioRef?.current` before writing the loudness accessor / speaking flag (and before clearing them on unmount). `RefObject.current` is nullable by type; the writes previously assumed it was always an object. - Add tests that pass an audioRef and assert it receives `getOutputVolume` and `speaking` while the agent speaks, and that both are cleared on unmount — the lip-sync loop reads straight from this ref and no prior test exercised the `if (audioRef)` branch. The third finding (assert rAF is never scheduled) is already covered: the `enabled: false` tests assert `requestAnimationFrame` is never called and that disabling mid-turn freezes the scheduled-frame count. The flagged line-44 test runs with `enabled: true` and a silent session, where the loop is meant to run, so asserting no frames there would be incorrect. Co-Authored-By: Claude Opus 4.8 --- .../human/RealtimeVoiceControls.test.tsx | 38 +++++++++++++++++-- .../features/human/RealtimeVoiceControls.tsx | 4 +- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/app/src/features/human/RealtimeVoiceControls.test.tsx b/app/src/features/human/RealtimeVoiceControls.test.tsx index 42e49245de..8bc1b76db8 100644 --- a/app/src/features/human/RealtimeVoiceControls.test.tsx +++ b/app/src/features/human/RealtimeVoiceControls.test.tsx @@ -6,12 +6,13 @@ */ import { configureStore } from '@reduxjs/toolkit'; import { fireEvent, render, screen } from '@testing-library/react'; -import type { ReactNode } from 'react'; +import type { ReactNode, RefObject } from 'react'; import { Provider } from 'react-redux'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import mascotReducer from '../../store/mascotSlice'; import RealtimeVoiceControls from './RealtimeVoiceControls'; +import type { RealtimeVoiceAudio } from './voice/amplitudeLipsync'; import type { RealtimeVoiceSession } from './voice/useRealtimeVoiceSession'; // `@elevenlabs/react`'s ConversationProvider is only a context shell here — pass @@ -49,11 +50,14 @@ const LABEL = { speaking: 'Speaking', } as const; -function renderControls(onSpeakingChange?: (speaking: boolean) => void) { +function renderControls( + onSpeakingChange?: (speaking: boolean) => void, + audioRef?: RefObject +) { const store = configureStore({ reducer: { mascot: mascotReducer } }); return render( - + ); } @@ -136,4 +140,32 @@ describe('RealtimeVoiceControls', () => { renderControls(onSpeakingChange); expect(onSpeakingChange).toHaveBeenLastCalledWith(false); }); + + // The lip-sync loop reads the SDK accessor and speaking flag straight out of + // this ref (see useAmplitudeLipsync); a regression that stopped publishing + // them would freeze the mascot's mouth mid-turn without failing any of the + // presentational tests above, so pin the wiring directly. + it('publishes the loudness accessor and speaking flag into audioRef', () => { + const getOutputVolume = () => 0.5; + session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume }); + const audioRef: RefObject = { + current: { getOutputVolume: null, speaking: false }, + }; + renderControls(undefined, audioRef); + expect(audioRef.current?.getOutputVolume).toBe(getOutputVolume); + expect(audioRef.current?.speaking).toBe(true); + }); + + it('clears the audioRef when the session ends (unmount)', () => { + session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume: () => 0.5 }); + const audioRef: RefObject = { + current: { getOutputVolume: null, speaking: false }, + }; + const { unmount } = renderControls(undefined, audioRef); + expect(audioRef.current?.speaking).toBe(true); + + unmount(); + expect(audioRef.current?.getOutputVolume).toBeNull(); + expect(audioRef.current?.speaking).toBe(false); + }); }); diff --git a/app/src/features/human/RealtimeVoiceControls.tsx b/app/src/features/human/RealtimeVoiceControls.tsx index ee26cd51a3..2a965de2a5 100644 --- a/app/src/features/human/RealtimeVoiceControls.tsx +++ b/app/src/features/human/RealtimeVoiceControls.tsx @@ -36,7 +36,7 @@ function RealtimeVoiceControlsInner({ const { getOutputVolume, isSpeaking } = session; const speaking = active && isSpeaking; useEffect(() => { - if (audioRef) { + if (audioRef?.current) { audioRef.current.getOutputVolume = active ? getOutputVolume : null; audioRef.current.speaking = speaking; } @@ -52,7 +52,7 @@ function RealtimeVoiceControlsInner({ // edge the page gates its loop on. useEffect( () => () => { - if (audioRef) { + if (audioRef?.current) { audioRef.current.getOutputVolume = null; audioRef.current.speaking = false; } From ca25bcafb4af06fa381f64823df3231eb68ab304 Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 14 Aug 2026 17:46:38 +0530 Subject: [PATCH 5/6] =?UTF-8?q?test(voice):=20cover=20the=20active?= =?UTF-8?q?=E2=86=92idle=20audioRef=20cleanup=20while=20mounted?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address CodeRabbit review on #5546: add a test for a session that ends while the control stays mounted (active → idle via rerender), asserting the live effect clears the ref — a path distinct from the unmount cleanup already covered. Rename the unmount test to name that path explicitly. Co-Authored-By: Claude Opus 4.8 --- .../human/RealtimeVoiceControls.test.tsx | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/app/src/features/human/RealtimeVoiceControls.test.tsx b/app/src/features/human/RealtimeVoiceControls.test.tsx index 8bc1b76db8..9a3d0a06d3 100644 --- a/app/src/features/human/RealtimeVoiceControls.test.tsx +++ b/app/src/features/human/RealtimeVoiceControls.test.tsx @@ -156,7 +156,7 @@ describe('RealtimeVoiceControls', () => { expect(audioRef.current?.speaking).toBe(true); }); - it('clears the audioRef when the session ends (unmount)', () => { + it('clears the audioRef on unmount cleanup', () => { session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume: () => 0.5 }); const audioRef: RefObject = { current: { getOutputVolume: null, speaking: false }, @@ -168,4 +168,31 @@ describe('RealtimeVoiceControls', () => { expect(audioRef.current?.getOutputVolume).toBeNull(); expect(audioRef.current?.speaking).toBe(false); }); + + // Ending a session doesn't unmount the control — the card stays on screen and + // the session hook just returns to 'idle'. The live effect (not the unmount + // cleanup) has to clear the ref on that transition, or the mascot would keep + // reading a stale accessor after the agent has gone. + it('clears the audioRef when the session goes idle while still mounted', () => { + const store = configureStore({ reducer: { mascot: mascotReducer } }); + const audioRef: RefObject = { + current: { getOutputVolume: null, speaking: false }, + }; + session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume: () => 0.5 }); + const { rerender } = render( + + + + ); + expect(audioRef.current?.speaking).toBe(true); + + session = makeSession({ state: 'idle', isSpeaking: false }); + rerender( + + + + ); + expect(audioRef.current?.getOutputVolume).toBeNull(); + expect(audioRef.current?.speaking).toBe(false); + }); }); From a970d514319f68a9f4955d1c15cc2026edd748be Mon Sep 17 00:00:00 2001 From: Shanu Date: Fri, 14 Aug 2026 17:56:15 +0530 Subject: [PATCH 6/6] test(voice): cover the isSpeaking edge while the session stays active MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address tinysweeper finding on #5546: add a test that flips isSpeaking true→false while the session remains 'active', asserting onSpeakingChange lands on false and audioRef.speaking clears while getOutputVolume stays published. This pins `speaking` in the publication effect's dep array — the first-render assertions would pass even if it were dropped, yet the mouth would freeze open mid-turn. Co-Authored-By: Claude Opus 4.8 --- .../human/RealtimeVoiceControls.test.tsx | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/app/src/features/human/RealtimeVoiceControls.test.tsx b/app/src/features/human/RealtimeVoiceControls.test.tsx index 9a3d0a06d3..2e399e7f00 100644 --- a/app/src/features/human/RealtimeVoiceControls.test.tsx +++ b/app/src/features/human/RealtimeVoiceControls.test.tsx @@ -195,4 +195,39 @@ describe('RealtimeVoiceControls', () => { expect(audioRef.current?.getOutputVolume).toBeNull(); expect(audioRef.current?.speaking).toBe(false); }); + + // The agent falling silent mid-session flips only `isSpeaking` — the session + // stays 'active'. The publication effect must re-run on that edge alone, or a + // stale `speaking: true` would sit in the ref and freeze the mascot's mouth + // open. Keeping `active` true here (unlike the goes-idle case above, where the + // active→idle change would re-run the effect regardless) pins `speaking` in + // the effect's dependency array specifically. + it('publishes the speaking edge while the session stays active', () => { + const store = configureStore({ reducer: { mascot: mascotReducer } }); + const onSpeakingChange = vi.fn(); + const getOutputVolume = () => 0.5; + const audioRef: RefObject = { + current: { getOutputVolume: null, speaking: false }, + }; + session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume }); + const { rerender } = render( + + + + ); + expect(onSpeakingChange).toHaveBeenLastCalledWith(true); + expect(audioRef.current?.speaking).toBe(true); + + session = makeSession({ state: 'active', isSpeaking: false, getOutputVolume }); + rerender( + + + + ); + expect(onSpeakingChange).toHaveBeenLastCalledWith(false); + expect(audioRef.current?.speaking).toBe(false); + // The session never closed, so the accessor stays published — only the + // speaking flag dropped. + expect(audioRef.current?.getOutputVolume).toBe(getOutputVolume); + }); });