diff --git a/app/src/features/human/HumanPage.tsx b/app/src/features/human/HumanPage.tsx index 08ea7642aa..7df53840ce 100644 --- a/app/src/features/human/HumanPage.tsx +++ b/app/src/features/human/HumanPage.tsx @@ -1,4 +1,4 @@ -import { useMemo } from 'react'; +import { useMemo, useRef, useState } from 'react'; import { useT } from '../../lib/i18n/I18nContext'; import { useAppDispatch, useAppSelector } from '../../store/hooks'; @@ -22,6 +22,8 @@ import { import { useMascotManifest } from './Mascot/manifest/useMascotManifest'; import RealtimeVoiceControls from './RealtimeVoiceControls'; import { useHumanMascot } from './useHumanMascot'; +import { IDLE_REALTIME_VOICE_AUDIO, type RealtimeVoiceAudio } from './voice/amplitudeLipsync'; +import { useAmplitudeLipsync } from './voice/useAmplitudeLipsync'; import { resolveHumanVoiceEntry } from './voiceEntry'; const HumanPage = () => { @@ -35,6 +37,24 @@ const HumanPage = () => { const speakReplies = useAppSelector(selectSpeakReplies); const { face, visemeCode } = useHumanMascot({ speakReplies }); + + // Lip-sync for the realtime voice session. The session lives inside + // RealtimeVoiceControls (which owns its own ConversationProvider), so it + // publishes its output-loudness accessor into this ref and the mascot samples + // it per frame — a 60fps signal must not travel through React state. + const realtimeAudioRef = useRef({ ...IDLE_REALTIME_VOICE_AUDIO }); + // The agent's speaking edge, lifted out of RealtimeVoiceControls so it can gate + // the lip-sync loop below. Flips a couple of times per turn, so it is cheap as + // state (the 60fps amplitude stays in the ref). While it is false — an idle + // realtime session, or the classic voice path that never mounts the control — + // the loop schedules no frames at all. + const [realtimeSpeaking, setRealtimeSpeaking] = useState(false); + const realtimeLipsync = useAmplitudeLipsync(realtimeAudioRef, realtimeSpeaking); + + // While the agent is speaking its own audio drives the mouth; otherwise the + // classic path keeps ownership, so the two never fight over the same frame. + const mascotFace = realtimeLipsync.active ? 'speaking' : face; + const mascotVisemeCode = realtimeLipsync.active ? realtimeLipsync.visemeCode : visemeCode; const mascotColor = useAppSelector(selectMascotColor); const customPrimary = useAppSelector(selectCustomPrimaryColor); const customSecondary = useAppSelector(selectCustomSecondaryColor); @@ -76,7 +96,14 @@ const HumanPage = () => { : null} + voiceChatControl={ + voiceEntry === 'realtime' ? ( + + ) : null + } showMicComposer={voiceEntry !== 'realtime'} projectThreadList /> @@ -97,23 +124,23 @@ const HumanPage = () => {
{customMascotGifUrl ? ( - + ) : mascotEntry ? ( ) : ( )} @@ -125,7 +152,10 @@ const HumanPage = () => { tap-and-speak rather than a second button stacked on it. */} {voiceEntry === 'both' && (
- +
)} diff --git a/app/src/features/human/RealtimeVoiceControls.test.tsx b/app/src/features/human/RealtimeVoiceControls.test.tsx index 18301a8079..2e399e7f00 100644 --- a/app/src/features/human/RealtimeVoiceControls.test.tsx +++ b/app/src/features/human/RealtimeVoiceControls.test.tsx @@ -6,12 +6,13 @@ */ import { configureStore } from '@reduxjs/toolkit'; import { fireEvent, render, screen } from '@testing-library/react'; -import type { ReactNode } from 'react'; +import type { ReactNode, RefObject } from 'react'; import { Provider } from 'react-redux'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import mascotReducer from '../../store/mascotSlice'; import RealtimeVoiceControls from './RealtimeVoiceControls'; +import type { RealtimeVoiceAudio } from './voice/amplitudeLipsync'; import type { RealtimeVoiceSession } from './voice/useRealtimeVoiceSession'; // `@elevenlabs/react`'s ConversationProvider is only a context shell here — pass @@ -31,6 +32,7 @@ function makeSession(overrides: Partial = {}): RealtimeVoi state: 'idle', isSpeaking: false, mode: 'listening', + getOutputVolume: () => 0, error: null, start, stop, @@ -48,11 +50,14 @@ const LABEL = { speaking: 'Speaking', } as const; -function renderControls() { +function renderControls( + onSpeakingChange?: (speaking: boolean) => void, + audioRef?: RefObject +) { const store = configureStore({ reducer: { mascot: mascotReducer } }); return render( - + ); } @@ -112,4 +117,117 @@ describe('RealtimeVoiceControls', () => { expect(stop).toHaveBeenCalledTimes(1); expect(start).not.toHaveBeenCalled(); }); + + // The speaking edge gates the mascot's lip-sync loop on the page + // (useAmplitudeLipsync's `enabled`), so it must reflect `active && isSpeaking`, + // not either half alone. + it('reports not-speaking to onSpeakingChange while idle', () => { + const onSpeakingChange = vi.fn(); + renderControls(onSpeakingChange); + expect(onSpeakingChange).toHaveBeenLastCalledWith(false); + }); + + it('reports speaking only when the session is active and the agent speaks', () => { + const onSpeakingChange = vi.fn(); + session = makeSession({ state: 'active', isSpeaking: true }); + renderControls(onSpeakingChange); + expect(onSpeakingChange).toHaveBeenLastCalledWith(true); + }); + + it('reports not-speaking when active but the agent is silent', () => { + const onSpeakingChange = vi.fn(); + session = makeSession({ state: 'active', isSpeaking: false }); + renderControls(onSpeakingChange); + expect(onSpeakingChange).toHaveBeenLastCalledWith(false); + }); + + // The lip-sync loop reads the SDK accessor and speaking flag straight out of + // this ref (see useAmplitudeLipsync); a regression that stopped publishing + // them would freeze the mascot's mouth mid-turn without failing any of the + // presentational tests above, so pin the wiring directly. + it('publishes the loudness accessor and speaking flag into audioRef', () => { + const getOutputVolume = () => 0.5; + session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume }); + const audioRef: RefObject = { + current: { getOutputVolume: null, speaking: false }, + }; + renderControls(undefined, audioRef); + expect(audioRef.current?.getOutputVolume).toBe(getOutputVolume); + expect(audioRef.current?.speaking).toBe(true); + }); + + it('clears the audioRef on unmount cleanup', () => { + session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume: () => 0.5 }); + const audioRef: RefObject = { + current: { getOutputVolume: null, speaking: false }, + }; + const { unmount } = renderControls(undefined, audioRef); + expect(audioRef.current?.speaking).toBe(true); + + unmount(); + expect(audioRef.current?.getOutputVolume).toBeNull(); + expect(audioRef.current?.speaking).toBe(false); + }); + + // Ending a session doesn't unmount the control — the card stays on screen and + // the session hook just returns to 'idle'. The live effect (not the unmount + // cleanup) has to clear the ref on that transition, or the mascot would keep + // reading a stale accessor after the agent has gone. + it('clears the audioRef when the session goes idle while still mounted', () => { + const store = configureStore({ reducer: { mascot: mascotReducer } }); + const audioRef: RefObject = { + current: { getOutputVolume: null, speaking: false }, + }; + session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume: () => 0.5 }); + const { rerender } = render( + + + + ); + expect(audioRef.current?.speaking).toBe(true); + + session = makeSession({ state: 'idle', isSpeaking: false }); + rerender( + + + + ); + expect(audioRef.current?.getOutputVolume).toBeNull(); + expect(audioRef.current?.speaking).toBe(false); + }); + + // The agent falling silent mid-session flips only `isSpeaking` — the session + // stays 'active'. The publication effect must re-run on that edge alone, or a + // stale `speaking: true` would sit in the ref and freeze the mascot's mouth + // open. Keeping `active` true here (unlike the goes-idle case above, where the + // active→idle change would re-run the effect regardless) pins `speaking` in + // the effect's dependency array specifically. + it('publishes the speaking edge while the session stays active', () => { + const store = configureStore({ reducer: { mascot: mascotReducer } }); + const onSpeakingChange = vi.fn(); + const getOutputVolume = () => 0.5; + const audioRef: RefObject = { + current: { getOutputVolume: null, speaking: false }, + }; + session = makeSession({ state: 'active', isSpeaking: true, getOutputVolume }); + const { rerender } = render( + + + + ); + expect(onSpeakingChange).toHaveBeenLastCalledWith(true); + expect(audioRef.current?.speaking).toBe(true); + + session = makeSession({ state: 'active', isSpeaking: false, getOutputVolume }); + rerender( + + + + ); + expect(onSpeakingChange).toHaveBeenLastCalledWith(false); + expect(audioRef.current?.speaking).toBe(false); + // The session never closed, so the accessor stays published — only the + // speaking flag dropped. + expect(audioRef.current?.getOutputVolume).toBe(getOutputVolume); + }); }); diff --git a/app/src/features/human/RealtimeVoiceControls.tsx b/app/src/features/human/RealtimeVoiceControls.tsx index 01836fefbd..2a965de2a5 100644 --- a/app/src/features/human/RealtimeVoiceControls.tsx +++ b/app/src/features/human/RealtimeVoiceControls.tsx @@ -1,9 +1,11 @@ import { ConversationProvider } from '@elevenlabs/react'; +import { type RefObject, useEffect } from 'react'; import Button from '../../components/ui/Button'; import { useT } from '../../lib/i18n/I18nContext'; import { useAppSelector } from '../../store/hooks'; import { selectEffectiveMascotVoiceId } from '../../store/mascotSlice'; +import type { RealtimeVoiceAudio } from './voice/amplitudeLipsync'; import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession'; /** @@ -12,7 +14,13 @@ import { useRealtimeVoiceSession } from './voice/useRealtimeVoiceSession'; * Wraps its own `ConversationProvider` (required by `@elevenlabs/react`) so it * stays self-contained and adds no context to the rest of the app. */ -function RealtimeVoiceControlsInner() { +function RealtimeVoiceControlsInner({ + audioRef, + onSpeakingChange, +}: { + audioRef?: RefObject; + onSpeakingChange?: (speaking: boolean) => void; +}) { const { t } = useT(); const voiceId = useAppSelector(selectEffectiveMascotVoiceId); const session = useRealtimeVoiceSession({ voiceId }); @@ -20,6 +28,39 @@ function RealtimeVoiceControlsInner() { const active = session.state === 'active'; const connecting = session.state === 'connecting'; + // Publish the output-loudness accessor for the mascot's lip-sync. Written into + // a ref rather than lifted into state because the mascot samples it once per + // animation frame; see `useAmplitudeLipsync`. The session lives under this + // component's own ConversationProvider, so this is the only place that can + // reach it. + const { getOutputVolume, isSpeaking } = session; + const speaking = active && isSpeaking; + useEffect(() => { + if (audioRef?.current) { + audioRef.current.getOutputVolume = active ? getOutputVolume : null; + audioRef.current.speaking = speaking; + } + // Surface the speaking edge so the page can gate the mascot's lip-sync rAF + // loop (`useAmplitudeLipsync`'s `enabled`) — an idle or classic Human tab + // then schedules no frames. Unlike the 60fps amplitude above, this flips a + // couple of times per turn, so it is cheap to lift into React state. + onSpeakingChange?.(speaking); + }, [audioRef, active, speaking, getOutputVolume, onSpeakingChange]); + + // A session that ends mid-speech would otherwise leave `speaking` true and the + // mouth frozen open — reset both the ref the mascot samples and the speaking + // edge the page gates its loop on. + useEffect( + () => () => { + if (audioRef?.current) { + audioRef.current.getOutputVolume = null; + audioRef.current.speaking = false; + } + onSpeakingChange?.(false); + }, + [audioRef, onSpeakingChange] + ); + const label = connecting ? t('voice.mode.connecting') : active @@ -51,10 +92,18 @@ function RealtimeVoiceControlsInner() { ); } -export default function RealtimeVoiceControls() { +export default function RealtimeVoiceControls({ + audioRef, + onSpeakingChange, +}: { + /** Optional sink for the mascot's lip-sync signal (see RealtimeVoiceAudio). */ + audioRef?: RefObject; + /** Notified with the agent's speaking edge so the page can gate the loop. */ + onSpeakingChange?: (speaking: boolean) => void; +}) { return ( - + ); } diff --git a/app/src/features/human/voice/amplitudeLipsync.test.ts b/app/src/features/human/voice/amplitudeLipsync.test.ts new file mode 100644 index 0000000000..4f8d88a3c6 --- /dev/null +++ b/app/src/features/human/voice/amplitudeLipsync.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; + +import { amplitudeToVisemeCode, smoothAmplitude } from './amplitudeLipsync'; + +describe('amplitudeToVisemeCode', () => { + it('rests the mouth on silence and room tone', () => { + expect(amplitudeToVisemeCode(0)).toBe('sil'); + // Below the floor: the tail of a word, not speech. Holding the mouth open + // through these gaps is what makes naive amplitude lip-sync look slack. + expect(amplitudeToVisemeCode(0.03)).toBe('sil'); + }); + + it('opens the mouth further as the signal gets louder', () => { + const codes = [0.08, 0.2, 0.6].map(amplitudeToVisemeCode); + expect(codes).toEqual(['I', 'E', 'aa']); + }); + + // A garbage reading (analyser torn down mid-frame) must not map to a wide-open + // mouth that then sticks for the rest of the call. Infinity rests for the same + // reason NaN does: it is a broken sample, not a loud one. + it('rests on a non-finite reading', () => { + expect(amplitudeToVisemeCode(Number.NaN)).toBe('sil'); + expect(amplitudeToVisemeCode(Number.POSITIVE_INFINITY)).toBe('sil'); + }); +}); + +describe('smoothAmplitude', () => { + it('moves toward the new sample', () => { + expect(smoothAmplitude(0, 1)).toBeGreaterThan(0); + expect(smoothAmplitude(1, 0)).toBeLessThan(1); + }); + + // Asymmetric on purpose: consonant onsets must land on time, but the mouth + // must not snap shut inside a word. + it('opens faster than it closes', () => { + const opening = smoothAmplitude(0, 1) - 0; + const closing = 1 - smoothAmplitude(1, 0); + expect(opening).toBeGreaterThan(closing); + }); + + it('converges on a held level rather than oscillating', () => { + let level = 0; + for (let i = 0; i < 40; i += 1) level = smoothAmplitude(level, 0.5); + expect(level).toBeCloseTo(0.5, 2); + }); +}); diff --git a/app/src/features/human/voice/amplitudeLipsync.ts b/app/src/features/human/voice/amplitudeLipsync.ts new file mode 100644 index 0000000000..0515a8e2b5 --- /dev/null +++ b/app/src/features/human/voice/amplitudeLipsync.ts @@ -0,0 +1,83 @@ +/** + * Amplitude-driven lip-sync for the realtime voice session (#5399). + * + * The classic tap-and-speak path animates the mouth from a viseme *timeline*: + * frames of `{viseme, ms}` sampled against our own audio element's clock. That + * is not available here — the realtime SDK owns playback, so there is no + * `currentMs()` to sample and no per-character timing unless we subscribe to + * alignment events and reconstruct one. + * + * What the SDK does expose is the output signal itself (`getOutputVolume()`), + * so the mouth is driven from loudness instead. That is genuinely less accurate + * — it opens and closes with the envelope rather than forming phonemes, so no + * `M`/`F` closures — but it is in sync *by construction*, because it is the + * audio being played rather than a prediction of it. A frozen mouth while the + * agent talks reads as broken; an approximate one reads as alive. + * + * The viseme-timeline version is the follow-up, and this is the fallback it + * would keep for the case where alignment is absent or has run dry. + */ + +/** Viseme codes, ordered by how open the mouth is. */ +const REST = 'sil'; +const NARROW = 'I'; // openness 0.30 +const MID = 'E'; // openness 0.45 +const OPEN = 'aa'; // openness ~0.95 + +/** + * Below this the signal is room tone or the tail of a word, not speech. Holding + * the mouth open through those gaps is what makes naive amplitude lip-sync look + * slack-jawed, so anything under it rests. + */ +const SILENCE_FLOOR = 0.04; + +/** Where the mouth steps from narrow to mid, and from mid to wide open. */ +const MID_THRESHOLD = 0.12; +const OPEN_THRESHOLD = 0.28; + +/** + * Smoothing applied to the raw reading, as the weight given to the new sample. + * + * `getOutputVolume()` is sampled per animation frame and is noisy at that rate: + * fed straight through it produces a chattering mouth that reads as a glitch + * rather than as speech. Asymmetric on purpose — opening tracks the signal + * quickly so consonant onsets land on time, closing lags so the mouth does not + * snap shut inside a word. + */ +const ATTACK = 0.55; +const RELEASE = 0.18; + +/** Smooth one amplitude sample toward the previous level. Pure + unit-tested. */ +export function smoothAmplitude(previous: number, sample: number): number { + const weight = sample > previous ? ATTACK : RELEASE; + return previous + (sample - previous) * weight; +} + +/** + * Map a smoothed amplitude (0..1) onto a viseme code. Steps rather than + * interpolates because the Rive mouth is driven by a code, not a scalar. + * Pure + unit-tested. + */ +export function amplitudeToVisemeCode(level: number): string { + if (!Number.isFinite(level) || level < SILENCE_FLOOR) return REST; + if (level < MID_THRESHOLD) return NARROW; + if (level < OPEN_THRESHOLD) return MID; + return OPEN; +} + +/** + * What the realtime controls publish for the mascot to read. Held in a ref and + * mutated in place: the mascot samples it once per animation frame, and routing + * a 60fps signal through React state would re-render the page on every frame. + */ +export interface RealtimeVoiceAudio { + /** SDK accessor for output loudness, or null when no session is live. */ + getOutputVolume: (() => number) | null; + /** Whether the agent is currently speaking (SDK `isSpeaking`). */ + speaking: boolean; +} + +export const IDLE_REALTIME_VOICE_AUDIO: RealtimeVoiceAudio = { + getOutputVolume: null, + speaking: false, +}; diff --git a/app/src/features/human/voice/useAmplitudeLipsync.test.ts b/app/src/features/human/voice/useAmplitudeLipsync.test.ts new file mode 100644 index 0000000000..0ebbdde83d --- /dev/null +++ b/app/src/features/human/voice/useAmplitudeLipsync.test.ts @@ -0,0 +1,160 @@ +import { act, renderHook } from '@testing-library/react'; +import { createRef } from 'react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { type RealtimeVoiceAudio } from './amplitudeLipsync'; +import { useAmplitudeLipsync } from './useAmplitudeLipsync'; + +/** Drive the rAF loop by hand so frames are deterministic. */ +let frames: FrameRequestCallback[] = []; +let raf: ReturnType; +let cancel: ReturnType; + +function flushFrames(count: number): void { + for (let i = 0; i < count; i += 1) { + const pending = frames; + frames = []; + act(() => pending.forEach(cb => cb(performance.now()))); + } +} + +function audioRef(overrides: Partial = {}) { + const ref = createRef() as { current: RealtimeVoiceAudio }; + ref.current = { getOutputVolume: null, speaking: false, ...overrides }; + return ref; +} + +describe('useAmplitudeLipsync', () => { + beforeEach(() => { + frames = []; + raf = vi.fn((cb: FrameRequestCallback) => { + frames.push(cb); + return frames.length; + }); + cancel = vi.fn(); + vi.stubGlobal('requestAnimationFrame', raf); + vi.stubGlobal('cancelAnimationFrame', cancel); + }); + afterEach(() => vi.unstubAllGlobals()); + + it('stays inactive and rested while nothing is speaking', () => { + const { result } = renderHook(() => useAmplitudeLipsync(audioRef(), true)); + flushFrames(3); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + }); + + it('drives the mouth from output loudness while the agent speaks', () => { + const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); + flushFrames(10); + expect(result.current.active).toBe(true); + expect(result.current.visemeCode).toBe('aa'); + }); + + // The mouth must return to rest when the turn ends, not freeze on its last + // shape — a stuck-open mascot is worse than one that never moved. + it('rests the mouth when speaking stops', () => { + const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); + flushFrames(10); + expect(result.current.visemeCode).toBe('aa'); + + ref.current.speaking = false; + flushFrames(2); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + }); + + // The SDK reads a live analyser; a session torn down mid-frame throws rather + // than returning 0. An uncaught throw would kill the loop and freeze the mouth. + it('survives an accessor that throws mid-session', () => { + const ref = audioRef({ + speaking: true, + getOutputVolume: () => { + throw new Error('analyser closed'); + }, + }); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); + expect(() => flushFrames(5)).not.toThrow(); + expect(result.current.visemeCode).toBe('sil'); + }); + + // Smoothing toward 0 would hold the mouth open for ~16 frames after the audio + // is already gone, so a bad reading resets on the very next frame. + it('rests immediately when the accessor throws after a loud sample', () => { + let volume = 0.9; + const ref = audioRef({ + speaking: true, + getOutputVolume: () => { + if (volume < 0) throw new Error('analyser closed'); + return volume; + }, + }); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); + flushFrames(10); + expect(result.current.visemeCode).toBe('aa'); + + volume = -1; // next call throws + flushFrames(1); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + }); + + // A non-finite sample smoothed into the level poisons it permanently: every + // later sample stays NaN and the mouth never animates again for the rest of + // the call. Guarding only at the viseme mapping hides that as a quiet mouth. + it('recovers on the next valid sample after a non-finite reading', () => { + let volume = 0.9; + const ref = audioRef({ speaking: true, getOutputVolume: () => volume }); + const { result } = renderHook(() => useAmplitudeLipsync(ref, true)); + flushFrames(10); + expect(result.current.visemeCode).toBe('aa'); + + volume = Number.NaN; + flushFrames(1); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + + // The level must not have been poisoned — real audio animates again. + volume = 0.9; + flushFrames(10); + expect(result.current).toEqual({ active: true, visemeCode: 'aa' }); + }); + + it('cancels its frame loop on unmount', () => { + const { unmount } = renderHook(() => useAmplitudeLipsync(audioRef(), true)); + unmount(); + expect(cancel).toHaveBeenCalled(); + }); + + // The loop is the whole cost of the feature, and it must not run when the tab + // is not driving a speaking session — a disabled hook schedules no frames at + // all, even with a live-looking accessor sitting in the ref (#5546). + it('schedules no frames while disabled', () => { + const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); + const { result } = renderHook(() => useAmplitudeLipsync(ref, false)); + flushFrames(3); + expect(raf).not.toHaveBeenCalled(); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + }); + + // Disabling mid-turn must tear the loop down, not just idle it: no further + // frames are queued, and the mouth resets rather than freezing on its last + // shape (the loop that would otherwise commit 'sil' is gone). + it('stops scheduling frames and rests when it becomes disabled', () => { + const ref = audioRef({ speaking: true, getOutputVolume: () => 0.9 }); + const { result, rerender } = renderHook(({ enabled }) => useAmplitudeLipsync(ref, enabled), { + initialProps: { enabled: true }, + }); + flushFrames(10); + expect(result.current.visemeCode).toBe('aa'); + const scheduledWhileActive = raf.mock.calls.length; + expect(scheduledWhileActive).toBeGreaterThan(0); + + rerender({ enabled: false }); + expect(cancel).toHaveBeenCalled(); + expect(result.current).toEqual({ active: false, visemeCode: 'sil' }); + + // No new frames after the idle transition — pins the regression the gate + // exists to prevent (the loop used to reschedule unconditionally). + flushFrames(5); + expect(raf.mock.calls.length).toBe(scheduledWhileActive); + }); +}); diff --git a/app/src/features/human/voice/useAmplitudeLipsync.ts b/app/src/features/human/voice/useAmplitudeLipsync.ts new file mode 100644 index 0000000000..d8fec91ebd --- /dev/null +++ b/app/src/features/human/voice/useAmplitudeLipsync.ts @@ -0,0 +1,108 @@ +import { type RefObject, useEffect, useRef, useState } from 'react'; + +import { + amplitudeToVisemeCode, + type RealtimeVoiceAudio, + smoothAmplitude, +} from './amplitudeLipsync'; + +export interface AmplitudeLipsync { + /** True while the realtime agent is speaking and driving the mouth. */ + active: boolean; + /** Viseme code for Rive's `mouthVisemeCode` input. */ + visemeCode: string; +} + +/** + * Drive the mascot's mouth from the realtime session's output loudness. + * + * The animation-frame loop exists only while `enabled` is true — the caller + * passes the agent's speaking edge, so an idle Human tab (and the classic + * voice path, which never speaks a realtime session) schedules no frames at + * all. `enabled` gates the effect rather than being read inside the loop + * because a ref cannot re-subscribe an effect: the on/off edge changes a couple + * of times per turn, so it is cheap as a dependency, while the amplitude it + * drives stays in a ref because that moves every frame. Reads the SDK accessor + * out of a ref rather than props because the session lives inside + * `RealtimeVoiceControls` (which owns its own `ConversationProvider`) — see + * `RealtimeVoiceAudio`. + * + * State is committed only when the viseme code actually changes. The smoothed + * level moves every frame, but the code it maps to steps between four values, + * so re-rendering on the raw level would reconcile the page ~60 times a second + * to produce the same mouth — the exact cost the chat panel is memoised to + * avoid (#5357). + */ +export function useAmplitudeLipsync( + audio: RefObject, + enabled: boolean +): AmplitudeLipsync { + const [state, setState] = useState({ active: false, visemeCode: 'sil' }); + const levelRef = useRef(0); + // Mirrors `state` for the loop to compare against without re-subscribing the + // effect on every change. + const codeRef = useRef('sil'); + const activeRef = useRef(false); + + useEffect(() => { + const commit = (active: boolean, visemeCode: string): void => { + if (active === activeRef.current && visemeCode === codeRef.current) return; + activeRef.current = active; + codeRef.current = visemeCode; + setState({ active, visemeCode }); + }; + + // Disabled: schedule nothing, and make sure the mouth is at rest. The loop + // is the only thing that commits 'sil' when speech stops, and it does not + // run here, so the reset has to happen on the disabling render instead — + // otherwise the mouth would freeze on its last shape. + if (!enabled) { + levelRef.current = 0; + commit(false, 'sil'); + return; + } + + let raf = 0; + let stopped = false; + + const tick = (): void => { + if (stopped) return; + const { getOutputVolume, speaking } = audio.current; + if (!speaking || !getOutputVolume) { + levelRef.current = 0; + commit(false, 'sil'); + } else { + // The SDK reads from a live analyser, so a session torn down mid-frame + // makes this throw rather than return 0. + let sample: number; + try { + sample = getOutputVolume(); + } catch { + sample = Number.NaN; + } + if (Number.isFinite(sample)) { + levelRef.current = smoothAmplitude(levelRef.current, sample); + commit(true, amplitudeToVisemeCode(levelRef.current)); + } else { + // A bad reading resets rather than decays, for two reasons. Smoothing + // toward 0 would hold the mouth open for ~16 frames after the audio is + // already gone; and smoothing toward a non-finite value poisons + // `levelRef` permanently — every later sample stays NaN, so the mouth + // never animates again for the rest of the call. Guarding only at the + // viseme mapping hides that as a quiet mouth instead of an error. + levelRef.current = 0; + commit(false, 'sil'); + } + } + raf = window.requestAnimationFrame(tick); + }; + + raf = window.requestAnimationFrame(tick); + return () => { + stopped = true; + window.cancelAnimationFrame(raf); + }; + }, [audio, enabled]); + + return state; +} diff --git a/app/src/features/human/voice/useRealtimeVoiceSession.ts b/app/src/features/human/voice/useRealtimeVoiceSession.ts index f6d355a721..87bd5614a5 100644 --- a/app/src/features/human/voice/useRealtimeVoiceSession.ts +++ b/app/src/features/human/voice/useRealtimeVoiceSession.ts @@ -31,6 +31,12 @@ export interface RealtimeVoiceSession { isSpeaking: boolean; /** ElevenLabs turn mode; `listening` while the user speaks. */ mode: 'speaking' | 'listening'; + /** + * Output loudness (0..1) of the agent's voice, sampled from the SDK's + * analyser. Drives the mascot's amplitude lip-sync — the realtime SDK owns + * playback, so this is the only signal available to animate the mouth against. + */ + getOutputVolume: () => number; error: string | null; /** Fetch a signed URL and open the WebSocket session. Idempotent while busy. */ start: () => Promise; @@ -174,6 +180,7 @@ export function useRealtimeVoiceSession(opts?: { voiceId?: string }): RealtimeVo state, isSpeaking: conversation.isSpeaking, mode: conversation.mode, + getOutputVolume: conversation.getOutputVolume, error, start, stop,