diff --git a/packages/chat/src/components/NewFooter/BottomLinks/index.tsx b/packages/chat/src/components/NewFooter/BottomLinks/index.tsx index ebd2740ef..5b21ec447 100644 --- a/packages/chat/src/components/NewFooter/BottomLinks/index.tsx +++ b/packages/chat/src/components/NewFooter/BottomLinks/index.tsx @@ -19,7 +19,10 @@ export const BottomLinks: React.FC = ({ extraLinkText, showPoweredBy, }) => { - const showExtraLink = extraLinkText && extraLinkUrl; + const showExtraLink = !!extraLinkText && !!extraLinkUrl; + const showExtraText = !!extraLinkText && !extraLinkUrl; + + const showSeparator = showPoweredBy && (showExtraLink || showExtraText); return (
@@ -31,9 +34,9 @@ export const BottomLinks: React.FC = ({
)} - {showPoweredBy && showExtraLink &&
} + {showSeparator &&
} - {extraLinkText && !extraLinkUrl &&
{extraLinkText}
} + {showExtraText &&
{extraLinkText}
} {showExtraLink && ( diff --git a/packages/chat/src/views/ChatWidget/ChatWidget.story.tsx b/packages/chat/src/views/ChatWidget/ChatWidget.story.tsx index afc616b2f..e592509c3 100644 --- a/packages/chat/src/views/ChatWidget/ChatWidget.story.tsx +++ b/packages/chat/src/views/ChatWidget/ChatWidget.story.tsx @@ -95,7 +95,7 @@ export const Base = { allowDangerousHTML: true, voice: { url: 'https://runtime-api-review-operator.us-2.development.voiceflow.com', - accessToken: 'VF.DM.67703df8c466f3697baf4df9.OY8scQUFwMJAt3c1', + accessToken: 'test', }, }} assistant={{ diff --git a/packages/chat/src/views/VoiceWidget/VoiceWidget.view.tsx b/packages/chat/src/views/VoiceWidget/VoiceWidget.view.tsx index 330efeb65..d71f23a76 100644 --- a/packages/chat/src/views/VoiceWidget/VoiceWidget.view.tsx +++ b/packages/chat/src/views/VoiceWidget/VoiceWidget.view.tsx @@ -5,7 +5,7 @@ import type { VoiceState } from '@/constant/voice.constant'; import { VOICE_STATE } from '@/constant/voice.constant'; import { RuntimeStateAPIContext } from '@/contexts'; -import { useVoiceController } from './hooks/use-voice-controller.hook'; +import { useVoiceService } from './hooks/use-voice-service.hook'; export const VoiceWidget = () => { const { assistant, config } = useContext(RuntimeStateAPIContext); @@ -15,14 +15,14 @@ export const VoiceWidget = () => { throw new Error('Voice is not configured in the config'); } - const voiceController = useVoiceController({ + const voiceService = useVoiceService({ url: config.voice.url, userID: config.userID, assistantID: config.verify.projectID, accessToken: config.voice.accessToken, }); - useEffect(() => voiceController.onStateUpdate((state) => setState(state)), [voiceController]); + useEffect(() => voiceService.onStateUpdate((state) => setState(state)), [voiceService]); return ( { footer={assistant.common.footerLink} settings={assistant.voice} poweredBy={assistant.common.poweredBy} - onEndCall={voiceController.endConversation} - onStartCall={voiceController.startConversation} + onEndCall={voiceService.endConversation} + onStartCall={voiceService.startConversation} /> ); }; diff --git a/packages/chat/src/views/VoiceWidget/hooks/Voice.controller.ts b/packages/chat/src/views/VoiceWidget/hooks/Voice.controller.ts deleted file mode 100644 index 64bc59801..000000000 --- a/packages/chat/src/views/VoiceWidget/hooks/Voice.controller.ts +++ /dev/null @@ -1,334 +0,0 @@ -import { VOICE_STATE, VoiceState } from '@/constant/voice.constant'; - -export interface VoiceControllerOptions { - url: string; - userID?: string; - accessToken: string; - assistantID: string; -} - -export class VoiceController { - url: string; - - state: string = VOICE_STATE.IDLE; - - userID = 'test'; - - listeners: Array<(state: VoiceState) => void> = []; - - accessToken: string; - - assistantID: string; - - webSocket: WebSocket | null = null; - - lastMarkIndex = 0; - - lastSentMarkIndex = 0; - - audioQueue = Promise.resolve(); - - talkingTimeout: NodeJS.Timeout | null = null; - - inputAudioStream: MediaStream | null = null; - - mediaRecorder: MediaRecorder | null = null; - - currentAudioElement: { - outputAudioElement: HTMLAudioElement; - outputSourceBuffer: SourceBuffer | null; - } = { - outputAudioElement: new Audio(), - outputSourceBuffer: null, - }; - - constructor({ url, userID, accessToken, assistantID }: VoiceControllerOptions) { - this.url = `${url}/voice/socket`; - this.userID = userID ?? this.userID; - this.assistantID = assistantID; - this.accessToken = accessToken; - } - - sendMark = () => { - if (this.lastMarkIndex === this.lastSentMarkIndex) return; - - this.lastSentMarkIndex = this.lastMarkIndex; - - this.webSocket?.send( - JSON.stringify({ - type: 'mark', - payload: { - markIndex: this.lastMarkIndex, - }, - }) - ); - }; - - base64ToArrayBuffer(base64: string) { - const binaryString = window.atob(base64); - const len = binaryString.length; - const bytes = new Uint8Array(len); - - for (let i = 0; i < len; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - - return bytes.buffer; - } - - waitForBufferUpdateEnd = (outputSourceBuffer: SourceBuffer, chunk: ArrayBuffer) => - new Promise((resolve, reject) => { - function onUpdateEnd() { - outputSourceBuffer.removeEventListener('updateend', onUpdateEnd); - outputSourceBuffer.removeEventListener('error', onError); - resolve(null); - } - - function onError(error: any) { - outputSourceBuffer.removeEventListener('updateend', onUpdateEnd); - outputSourceBuffer.removeEventListener('error', onError); - reject(error); - } - - outputSourceBuffer.addEventListener('updateend', onUpdateEnd); - outputSourceBuffer.addEventListener('error', onError); - - outputSourceBuffer.appendBuffer(chunk); - }); - - async handleIncomingChunk(base64Chunk: string, markIndex: number) { - const { outputSourceBuffer } = this.currentAudioElement; - - if (!outputSourceBuffer) return; - - this.audioQueue = this.audioQueue.then(async () => { - const chunkBuffer = this.base64ToArrayBuffer(base64Chunk); - - await this.waitForBufferUpdateEnd(outputSourceBuffer, chunkBuffer); - - // 3. Append buffer - if (markIndex > this.lastMarkIndex) { - this.lastMarkIndex = markIndex; - } - - return undefined; - }); - } - - clearAudio() { - this.currentAudioElement.outputSourceBuffer?.abort(); - this.currentAudioElement.outputAudioElement.pause(); - this.currentAudioElement.outputAudioElement.currentTime = 0; - this.currentAudioElement.outputAudioElement.src = ''; - - this.sendMark(); - this.currentAudioElement = this.createAudioElement(); - } - - createAudioElement() { - const outputAudioElement = new Audio(); - - const ref: { - outputAudioElement: HTMLAudioElement; - outputSourceBuffer: SourceBuffer | null; - } = { - outputAudioElement, - outputSourceBuffer: null, - }; - - const outputMediaSource = new MediaSource(); - - outputAudioElement.src = URL.createObjectURL(outputMediaSource); - - // Flag to track if we're initialized - let sourceOpen = false; - - // Called when outputMediaSource is ready to accept data. - function onSourceOpen() { - if (!sourceOpen) { - sourceOpen = true; - ref.outputSourceBuffer = outputMediaSource.addSourceBuffer('audio/mpeg'); - - // (Optional) Set the mode to 'segments' - // outputSourceBuffer.mode = 'segments'; - - console.info('SourceBuffer created: audio/mpeg'); - } - } - - // Listen for MediaSource to be ready - outputMediaSource.addEventListener('sourceopen', onSourceOpen); - outputAudioElement.addEventListener('waiting', this.sendMark); - outputAudioElement.addEventListener('stalled', this.sendMark); - outputAudioElement.addEventListener('timeupdate', () => { - console.log('timeupdate', outputAudioElement.currentTime, this.state); - - if (this.state !== VOICE_STATE.TALKING && outputAudioElement === this.currentAudioElement.outputAudioElement) { - this.updateState(VOICE_STATE.TALKING); - } - - if (this.talkingTimeout) { - clearTimeout(this.talkingTimeout); - } - - this.talkingTimeout = setTimeout(() => { - this.updateState(VOICE_STATE.LISTENING); - this.talkingTimeout = null; - }, 500); - }); - - outputAudioElement.play(); - - return ref; - } - - async startInputStream() { - try { - this.inputAudioStream = await navigator.mediaDevices.getUserMedia({ - audio: true, - }); - } catch (e) { - return console.error('Microphone access denied.', e); - } - - console.info('Got audio stream:', this.inputAudioStream); - - this.mediaRecorder = new MediaRecorder(this.inputAudioStream); - - console.info('Got media recorder:', this.mediaRecorder); - - this.mediaRecorder.addEventListener('dataavailable', (event) => { - if (event.data.size > 0 && this.webSocket && this.webSocket.readyState === WebSocket.OPEN) { - this.webSocket.send(event.data); - } - }); - - this.mediaRecorder.addEventListener('error', (event) => { - console.error('MediaRecorder error:', event.error); - }); - - this.mediaRecorder.start(500); - console.info('Started audio streaming...'); - - return this.inputAudioStream; - } - - stopInputStream() { - if (this.mediaRecorder && this.mediaRecorder.state !== 'inactive') { - this.mediaRecorder.stop(); - } - if (this.inputAudioStream) { - this.inputAudioStream.getTracks().forEach((track) => track.stop()); - } - - console.info('Stopped audio streaming.'); - } - - initWebSocket = () => - new Promise((resolve) => { - if (this.webSocket && this.webSocket.readyState === WebSocket.OPEN) { - return; - } - - this.webSocket = new WebSocket(this.url); - - this.webSocket.onopen = () => { - console.info('WebSocket opened!'); - resolve({}); - }; - - this.webSocket.onmessage = (event) => { - // If we get a text message, treat as text/JSON - const message = JSON.parse(event.data); - - console.info(event.data, message); - - if (message.type === 'audio') { - this.handleIncomingChunk(message.payload.audio, message.payload.markIndex); - } else if (message.type === 'end') { - console.info('Conversation ended by server.', 'system'); - this.stopInputStream(); - } else if (message.type === 'interrupt') { - this.clearAudio(); - } - }; - - this.webSocket.onclose = () => { - console.info('WebSocket disconnected!', 'system'); - }; - - this.webSocket.onerror = (err) => { - console.error('WebSocket error:', err); - }; - }); - - async start() { - this.updateState(VOICE_STATE.INITIALIZING); - - await this.initWebSocket(); - - this.currentAudioElement = this.createAudioElement(); - - const messageObj = { - type: 'start', - payload: { - userID: this.userID, - assistantID: this.assistantID, - authorization: this.accessToken, - }, - }; - - this.webSocket?.send(JSON.stringify(messageObj)); - - await this.startInputStream(); - - console.log('listening'); - - this.updateState(VOICE_STATE.LISTENING); - } - - end() { - this.stopInputStream(); - - this.audioQueue = Promise.resolve(); - this.webSocket?.close(); - this.webSocket = null; - this.lastMarkIndex = 0; - this.lastSentMarkIndex = 0; - this.inputAudioStream = null; - this.mediaRecorder = null; - this.currentAudioElement = { - outputAudioElement: new Audio(), - outputSourceBuffer: null, - }; - - if (this.talkingTimeout) { - clearTimeout(this.talkingTimeout); - this.talkingTimeout = null; - } - - this.updateState(VOICE_STATE.ENDED); - } - - startConversation = () => { - this.start(); - }; - - endConversation = () => { - this.end(); - }; - - updateState = (state: VoiceState) => { - this.state = state; - - this.listeners.forEach((listener) => listener(state)); - }; - - onStateUpdate = (cb: (state: VoiceState) => void) => { - this.listeners.push(cb); - - return () => { - this.listeners = this.listeners.filter((listener) => listener !== cb); - }; - }; -} diff --git a/packages/chat/src/views/VoiceWidget/hooks/use-audio-stream.hook.ts b/packages/chat/src/views/VoiceWidget/hooks/use-audio-stream.hook.ts deleted file mode 100644 index 14a75c395..000000000 --- a/packages/chat/src/views/VoiceWidget/hooks/use-audio-stream.hook.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { useCallback, useState } from 'react'; - -export const STREAM_STATE = { - IDLE: 'IDLE', - INITIALIZING: 'INITIALIZING', - ACTIVE: 'ACTIVE', - STOPPED: 'STOPPED', -}; - -type STREAM_STATE = (typeof STREAM_STATE)[keyof typeof STREAM_STATE]; - -export interface IUseAudioStream { - onReady?: (stream: MediaStream) => void; -} - -export const useAudioStream = (options?: IUseAudioStream) => { - const { onReady } = options ?? {}; - - const [stream, setStream] = useState(null); - const [error, setError] = useState<{ code: number; message: string; error: any } | null>(null); - const [state, setState] = useState(STREAM_STATE.IDLE); - - const initStream = useCallback(async () => { - try { - setError(null); - setState(STREAM_STATE.INITIALIZING); - - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - - setState(STREAM_STATE.ACTIVE); - setStream(stream); - - onReady?.(stream); - } catch (error) { - setState(STREAM_STATE.STOPPED); - setError({ - code: 0, - error, - message: 'Error initializing audio stream', - }); - } - }, [setError, setState, setStream]); - - const closeStream = useCallback(() => { - stream?.getTracks().forEach((track) => track.stop()); - - setStream(null); - setState(STREAM_STATE.STOPPED); - }, [stream]); - - return { - error, - state, - stream, - - initStream, - closeStream, - }; -}; diff --git a/packages/chat/src/views/VoiceWidget/hooks/use-voice-controller.hook.ts b/packages/chat/src/views/VoiceWidget/hooks/use-voice-controller.hook.ts deleted file mode 100644 index 0d3387c9d..000000000 --- a/packages/chat/src/views/VoiceWidget/hooks/use-voice-controller.hook.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { useCreateConst } from '@/hooks/cache.hook'; - -import type { VoiceControllerOptions } from './Voice.controller'; -import { VoiceController } from './Voice.controller'; - -interface IUserVoiceController extends VoiceControllerOptions{ -} - -export const useVoiceController = (options: IUserVoiceController) => { - const voiceController = useCreateConst(() => new VoiceController(options)); - - return voiceController; -}; diff --git a/packages/chat/src/views/VoiceWidget/hooks/use-voice-service.hook.ts b/packages/chat/src/views/VoiceWidget/hooks/use-voice-service.hook.ts new file mode 100644 index 000000000..2c4c3ee5a --- /dev/null +++ b/packages/chat/src/views/VoiceWidget/hooks/use-voice-service.hook.ts @@ -0,0 +1,8 @@ +import { useCreateConst } from '@/hooks/cache.hook'; + +import type { VoiceServiceOptions } from '../services/voice.service'; +import { VoiceService } from '../services/voice.service'; + +interface IUserVoiceService extends VoiceServiceOptions {} + +export const useVoiceService = (options: IUserVoiceService) => useCreateConst(() => new VoiceService(options)); diff --git a/packages/chat/src/views/VoiceWidget/hooks/use-websocket.hook.ts b/packages/chat/src/views/VoiceWidget/hooks/use-websocket.hook.ts deleted file mode 100644 index 0d626a693..000000000 --- a/packages/chat/src/views/VoiceWidget/hooks/use-websocket.hook.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { useCallback, useState } from 'react'; - -export const WEBSOCKET_STATE = { - IDLE: 'IDLE', - CONNECTING: 'CONNECTING', - OPEN: 'OPEN', - CLOSING: 'CLOSING', - CLOSED: 'CLOSED', -}; - -type WEBSOCKET_STATE = (typeof WEBSOCKET_STATE)[keyof typeof WEBSOCKET_STATE]; - -export interface IUseAudioStream { - url: string; - onOpened?: (ws: WebSocket) => void; -} - -export const useWebsocket = ({ url, onOpened }: IUseAudioStream) => { - const [webSocket, setWebSocket] = useState(null); - const [error, setError] = useState<{ code: number; message: string; error: any } | null>(null); - const [state, setState] = useState(WEBSOCKET_STATE.IDLE); - - const onClose = useCallback(() => { - webSocket?.removeEventListener('close', onClose); - - setState(WEBSOCKET_STATE.CLOSED); - setWebSocket(null); - }, [setState]); - - const initializeWebSocket = useCallback(async () => { - try { - setError(null); - setState(WEBSOCKET_STATE.CONNECTING); - - const webSocket = await new Promise((resolve, reject) => { - const ws = new WebSocket(url); - - const onOpen = () => { - removeListeners(); - resolve(ws); - }; - - const onClose = () => { - removeListeners(); - reject(new Error('WebSocket closed')); - }; - - const onError = (error: any) => { - removeListeners(); - reject(error); - }; - - ws.addEventListener('open', onOpen); - ws.addEventListener('close', onClose); - ws.addEventListener('error', onError); - - const removeListeners = () => { - ws.removeEventListener('open', onOpen); - ws.removeEventListener('close', onClose); - ws.removeEventListener('error', onError); - }; - }); - - webSocket.addEventListener('close', onClose); - - setState(WEBSOCKET_STATE.OPEN); - setWebSocket(webSocket); - - onOpened?.(webSocket); - } catch (error) { - setState(WEBSOCKET_STATE.CLOSED); - setError({ - code: 0, - error, - message: 'Error initializing websocket', - }); - } - }, [setError, setState, onClose, setWebSocket]); - - const closeWebSocket = useCallback(() => { - setState(WEBSOCKET_STATE.CLOSING); - - const closeWebSocket = () => { - webSocket?.removeEventListener('close', closeWebSocket); - setWebSocket(null); - setState(WEBSOCKET_STATE.CLOSED); - }; - - webSocket?.addEventListener('close', closeWebSocket); - webSocket?.close(); - }, [webSocket]); - - return { - error, - state, - webSocket, - - initializeWebSocket, - closeWebSocket, - }; -}; diff --git a/packages/chat/src/views/VoiceWidget/services/audio.service.ts b/packages/chat/src/views/VoiceWidget/services/audio.service.ts new file mode 100644 index 000000000..1c9165142 --- /dev/null +++ b/packages/chat/src/views/VoiceWidget/services/audio.service.ts @@ -0,0 +1,182 @@ +interface QueueItem { + markIndex: number; + arrayBuffer: ArrayBuffer; +} + +export class AudioService { + private audio: HTMLAudioElement; + + private stopped = false; + + private mediaSource: MediaSource | null = null; + + private lastMarkIndex = 0; + + private lastSentMarkIndex = 0; + + private onMark: (markIndex: number) => void; + + private onTalking: () => void; + + private onListening: () => void; + + private queue: QueueItem[] = []; + + private activeItem: QueueItem | null = null; + + private sourceBuffer: SourceBuffer | null = null; + + constructor(options: { onMark: (markIndex: number) => void; onListening: () => void; onTalking: () => void }) { + this.onMark = options.onMark; + this.onTalking = options.onTalking; + this.onListening = options.onListening; + + this.audio = new Audio(); + this.audio.addEventListener('ended', this.onAudioEnded); + this.audio.addEventListener('waiting', this.onAudioWaiting); + this.audio.addEventListener('stalled', this.onAudioStalled); + this.audio.addEventListener('playing', this.onAudioPlaying); + } + + private onAudioWaiting = () => { + this.sendMark(); + this.onListening(); + }; + + private onAudioStalled = () => { + this.sendMark(); + this.onListening(); + }; + + private onAudioPlaying = () => { + this.onTalking(); + }; + + private onAudioEnded = () => { + this.onListening(); + }; + + private sendMark() { + if (this.lastMarkIndex === this.lastSentMarkIndex) return; + + this.lastSentMarkIndex = this.lastMarkIndex; + + this.onMark(this.lastMarkIndex); + } + + private base64ToArrayBuffer(base64: string) { + const binaryString = window.atob(base64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + return bytes.buffer; + } + + private onBufferUpdated(markIndex: number) { + this.activeItem = null; + + if (markIndex > this.lastMarkIndex) { + this.lastMarkIndex = markIndex; + } + + if (this.queue.length) { + this.playQueue(); + } else { + this.sendMark(); + } + } + + private async updateSourceBuffer(markIndex: number, arrayBuffer: ArrayBuffer) { + await this.startAudio(); + + this.sourceBuffer?.addEventListener('error', () => this.onBufferUpdated(markIndex), { once: true }); + this.sourceBuffer?.addEventListener('updateend', () => this.onBufferUpdated(markIndex), { once: true }); + + this.sourceBuffer?.appendBuffer(arrayBuffer); + } + + private async playQueue() { + if (this.stopped || this.activeItem || !this.queue.length) return; + + this.activeItem = this.queue.shift()!; + + this.updateSourceBuffer(this.activeItem.markIndex, this.activeItem.arrayBuffer); + } + + addChunk(base64Chunk: string, markIndex: number) { + if (this.stopped) return; + + const arrayBuffer = this.base64ToArrayBuffer(base64Chunk); + + this.queue.push({ markIndex, arrayBuffer }); + this.playQueue(); + } + + private stopAudio() { + if (this.stopped) return; + + this.sourceBuffer?.abort(); + this.sourceBuffer = null; + + if (this.mediaSource?.readyState === 'open') { + this.mediaSource?.endOfStream(); + } + this.mediaSource = null; + + URL.revokeObjectURL(this.audio.src); + + this.audio.pause(); + this.audio.currentTime = 0; + this.audio.src = ''; + } + + private startAudio() { + if (this.stopped || this.mediaSource) return Promise.resolve(); + + return new Promise((resolve) => { + const mediaSource = new MediaSource(); + + mediaSource.addEventListener( + 'sourceopen', + () => { + this.sourceBuffer = mediaSource.addSourceBuffer('audio/mpeg'); + + resolve(); + }, + { once: true } + ); + + this.audio.src = URL.createObjectURL(mediaSource); + + this.audio.play(); + this.mediaSource = mediaSource; + }); + } + + interrupt() { + if (this.stopped) return; + + this.queue = []; + this.activeItem = null; + + this.stopAudio(); + + this.sendMark(); + + this.startAudio(); + } + + stop() { + this.stopAudio(); + + this.queue = []; + this.stopped = true; + this.activeItem = null; + this.lastMarkIndex = 0; + this.lastSentMarkIndex = 0; + } +} diff --git a/packages/chat/src/views/VoiceWidget/services/recorder.service.ts b/packages/chat/src/views/VoiceWidget/services/recorder.service.ts new file mode 100644 index 000000000..adba208fc --- /dev/null +++ b/packages/chat/src/views/VoiceWidget/services/recorder.service.ts @@ -0,0 +1,85 @@ +export class RecorderService { + private mediaRecorder: MediaRecorder | null = null; + + private inputAudioStream: MediaStream | null = null; + + private createAudioStreamPromise: Promise | null = null; + + constructor(private readonly onDataAvailable: (data: Blob) => void) {} + + private createAudioStream() { + if (this.createAudioStreamPromise) { + console.info('Returning existing input audio stream promise.'); + + return this.createAudioStreamPromise; + } + + if (this.inputAudioStream) { + return Promise.resolve(this.inputAudioStream); + } + + this.createAudioStreamPromise = navigator.mediaDevices + .getUserMedia({ audio: true }) + .catch((err) => { + console.error('Microphone access denied.', err); + + throw err; + }) + .finally(() => { + this.createAudioStreamPromise = null; + }); + + return this.createAudioStreamPromise; + } + + private onMediaRecorderError = (event: Event) => { + console.error('MediaRecorder error:', (event as ErrorEvent).error); + }; + + private onMediaRecorderDataAvailable = (event: BlobEvent) => { + console.info('Got data:', event.data); + + if (event.data.size > 0) { + this.onDataAvailable(event.data); + } + }; + + async start() { + if (this.inputAudioStream) { + console.info('Audio stream already exists, reusing.'); + + return; + } + + this.inputAudioStream = await this.createAudioStream(); + + console.info('Got audio stream:', this.inputAudioStream); + + this.mediaRecorder = new MediaRecorder(this.inputAudioStream); + + console.info('Got media recorder:', this.mediaRecorder); + + this.mediaRecorder.addEventListener('error', this.onMediaRecorderError); + this.mediaRecorder.addEventListener('dataavailable', this.onMediaRecorderDataAvailable); + + this.mediaRecorder?.start(1000); + + console.info('Started audio streaming...'); + } + + stop() { + if (this.mediaRecorder?.state !== 'inactive') { + this.mediaRecorder?.stop(); + } + + this.inputAudioStream?.getTracks().forEach((track) => track.stop()); + + this.mediaRecorder?.removeEventListener('error', this.onMediaRecorderError); + this.mediaRecorder?.removeEventListener('dataavailable', this.onMediaRecorderDataAvailable); + + this.mediaRecorder = null; + this.inputAudioStream = null; + + console.info('Stopped audio streaming.'); + } +} diff --git a/packages/chat/src/views/VoiceWidget/services/socket.service.ts b/packages/chat/src/views/VoiceWidget/services/socket.service.ts new file mode 100644 index 000000000..df65b9daf --- /dev/null +++ b/packages/chat/src/views/VoiceWidget/services/socket.service.ts @@ -0,0 +1,143 @@ +interface SocketAudioInputMessage { + type: 'audio'; + payload: { audio: string; markIndex: number }; +} + +interface SocketEndInputMessage { + type: 'end'; +} + +interface SocketInterruptInputMessage { + type: 'interrupt'; +} + +export type SocketInputMessage = SocketAudioInputMessage | SocketEndInputMessage | SocketInterruptInputMessage; + +interface SocketMarkOutputMessage { + type: 'mark'; + payload: { markIndex: number }; +} + +interface SocketStartOutputMessage { + type: 'start'; + payload: { userID: string; assistantID: string; authorization: string }; +} + +export type SocketOutputMessage = Blob | SocketMarkOutputMessage | SocketStartOutputMessage; + +export class SocketService { + private url: string; + + private socket: WebSocket | null = null; + + private onError: (error: unknown) => void; + + private onMessage: (message: SocketInputMessage) => void; + + private connectionPromise: Promise | null = null; + + constructor(options: { + url: string; + onError: (error: unknown) => void; + onMessage: (message: SocketInputMessage) => void; + }) { + this.url = options.url; + this.onError = options.onError; + this.onMessage = options.onMessage; + } + + private createSocket() { + if (this.connectionPromise) { + return this.connectionPromise; + } + + if (this.socket) { + return Promise.resolve(this.socket); + } + + this.connectionPromise = new Promise((resolve, reject) => { + const socket = new WebSocket(this.url); + + const onOpen = () => { + console.info('Socket connection established.'); + + removeListeners(); + resolve(socket); + }; + + const onError = (error: unknown) => { + console.error('Socket connection error.', error); + + removeListeners(); + reject(error); + }; + + socket.addEventListener('open', onOpen); + socket.addEventListener('error', onError); + + const removeListeners = () => { + socket?.removeEventListener('open', onOpen); + socket?.removeEventListener('error', onError); + }; + }).finally(() => { + this.connectionPromise = null; + }); + + return this.connectionPromise; + } + + private onSocketError = (event: Event) => { + console.error('socket error:', event); + + this.onError(event); + this.stop(); + }; + + private onSocketMessage = (event: MessageEvent) => { + const message = JSON.parse(event.data); + + console.info('socket message:', message); + + this.onMessage(message); + }; + + async start() { + if (this.socket) { + console.info('Socket already exists, reusing.'); + + return; + } + + this.socket = await this.createSocket(); + + this.socket.addEventListener('error', this.onSocketError); + this.socket.addEventListener('message', this.onSocketMessage); + } + + stop() { + if (this.socket?.readyState !== WebSocket.CLOSED && this.socket?.readyState !== WebSocket.CLOSING) { + this.socket?.close(); + } + + this.socket?.removeEventListener('error', this.onSocketError); + this.socket?.removeEventListener('message', this.onSocketMessage); + + this.socket = null; + } + + send(message: SocketOutputMessage) { + if (!this.socket) { + console.warn('Socket is not open, cannot send message.'); + + return; + } + + if (this.socket.readyState !== WebSocket.OPEN) { + console.warn('Socket is not open, cannot send message.'); + + return; + } + + this.socket.send(message instanceof Blob ? message : JSON.stringify(message)); + } +} diff --git a/packages/chat/src/views/VoiceWidget/services/voice.service.ts b/packages/chat/src/views/VoiceWidget/services/voice.service.ts new file mode 100644 index 000000000..8f640e301 --- /dev/null +++ b/packages/chat/src/views/VoiceWidget/services/voice.service.ts @@ -0,0 +1,131 @@ +import { VOICE_STATE, VoiceState } from '@/constant/voice.constant'; + +import { AudioService } from './audio.service'; +import { RecorderService } from './recorder.service'; +import { SocketInputMessage, SocketService } from './socket.service'; + +export interface VoiceServiceOptions { + url: string; + userID?: string; + accessToken: string; + assistantID: string; +} + +export class VoiceService { + private state: string = VOICE_STATE.IDLE; + + private userID: string; + + private listeners: Array<(state: VoiceState) => void> = []; + + private accessToken: string; + + private assistantID: string; + + private audio: AudioService | null = null; + + private socket: SocketService; + + private recorder: RecorderService; + + constructor({ url, userID, accessToken, assistantID }: VoiceServiceOptions) { + this.userID = userID ?? 'test'; + this.assistantID = assistantID; + this.accessToken = accessToken; + + this.socket = new SocketService({ + url: `${url}/voice/socket`, + onError: this.onSocketError, + onMessage: this.onSocketMessage, + }); + this.recorder = new RecorderService(this.onRecorderDataAvailable); + } + + private onAudioMark = (markIndex: number) => { + this.socket?.send({ type: 'mark', payload: { markIndex } }); + }; + + private onAudioTalking = () => { + this.updateState(VOICE_STATE.TALKING); + }; + + private onAudioListening = () => { + this.updateState(VOICE_STATE.LISTENING); + }; + + private onRecorderDataAvailable = (data: Blob) => { + this.socket.send(data); + }; + + private onSocketError = () => { + // TODO: Handle socket error + }; + + private onSocketMessage = (message: SocketInputMessage) => { + if (message.type === 'audio') { + this.audio?.addChunk(message.payload.audio, message.payload.markIndex); + } else if (message.type === 'end') { + console.info('Conversation ended by server.', 'system'); + this.recorder.stop(); + } else if (message.type === 'interrupt') { + this.audio?.interrupt(); + } + }; + + private async start() { + this.updateState(VOICE_STATE.INITIALIZING); + + this.audio = new AudioService({ + onMark: this.onAudioMark, + onTalking: this.onAudioTalking, + onListening: this.onAudioListening, + }); + + await this.socket.start(); + + this.socket.send({ + type: 'start', + payload: { + userID: this.userID, + assistantID: this.assistantID, + authorization: this.accessToken, + }, + }); + + await this.recorder.start(); + + this.updateState(VOICE_STATE.LISTENING); + + console.info('listening...'); + } + + private stop() { + this.recorder.stop(); + this.socket.stop(); + this.audio?.stop(); + + this.updateState(VOICE_STATE.ENDED); + } + + endConversation = () => this.stop(); + + startConversation = () => this.start(); + + updateState = (state: VoiceState) => { + if (this.state === state) return; + + this.state = state; + + this.listeners.forEach((listener) => listener(state)); + }; + + onStateUpdate = (cb: (state: VoiceState) => void) => { + this.listeners.push(cb); + + // Returns an unsubscribe function that removes this callback from listeners + // when called, preventing memory leaks and unwanted updates + return () => { + this.listeners = this.listeners.filter((listener) => listener !== cb); + }; + }; +}