From 2788d37888f19c436bbd59fb841778aaec69b0cc Mon Sep 17 00:00:00 2001 From: Vachhani-Tapan Date: Sun, 30 Aug 2026 13:34:03 +0530 Subject: [PATCH] feat(audio): implement foundational full-duplex WebSocket audio pipeline for STT/TTS (fixes #2278) --- .../controllers/audioPipelineController.js | 38 +++++++++ backend/server.js | 17 ++++ frontend/src/hooks/useAudioPipeline.js | 80 +++++++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 backend/controllers/audioPipelineController.js create mode 100644 frontend/src/hooks/useAudioPipeline.js diff --git a/backend/controllers/audioPipelineController.js b/backend/controllers/audioPipelineController.js new file mode 100644 index 00000000..0006ff9f --- /dev/null +++ b/backend/controllers/audioPipelineController.js @@ -0,0 +1,38 @@ +// audioPipelineController.js +// Handles full-duplex WebSocket connections for STT/LLM/TTS streaming. + +const setupAudioPipeline = (wss) => { + wss.on('connection', (ws) => { + console.log('New Audio Pipeline WebSocket Connection Established'); + + // In a real implementation, we would pipe these chunks to a streaming STT API + // (e.g., Whisper, Deepgram, Google Speech-to-Text). + ws.on('message', (message) => { + // 1. Receive binary audio chunks from the frontend client. + // 2. STT Provider converts to text. + + // MOCK: Echo back a mock transcript and simulated audio chunk + const mockTranscript = "That's a great question regarding system design."; + + // Send the text transcript back to the frontend for UI display + ws.send(JSON.stringify({ type: 'transcript', text: mockTranscript })); + + // 3. The transcript is passed to the LLM (e.g., OpenAI, Gemini) to generate an AI response. + // 4. The LLM response is passed to a TTS Provider (e.g., ElevenLabs) to generate audio. + + // MOCK: Send simulated binary audio bytes back to the client + const mockAudioBytes = Buffer.from('mock_audio_bytes_representing_speech'); + ws.send(mockAudioBytes); + }); + + ws.on('close', () => { + console.log('Audio Pipeline WebSocket Connection Closed'); + }); + + ws.on('error', (error) => { + console.error('Audio Pipeline WebSocket Error:', error); + }); + }); +}; + +module.exports = { setupAudioPipeline }; diff --git a/backend/server.js b/backend/server.js index 96db4300..0c473f93 100644 --- a/backend/server.js +++ b/backend/server.js @@ -12,6 +12,8 @@ const googleCalendarRoutes = require("./routes/googleCalendarRoutes"); const path = require("path"); const connectDB = require("./config/db"); const cookieParser = require("cookie-parser"); +const WebSocket = require("ws"); +const { setupAudioPipeline } = require("./controllers/audioPipelineController"); const { generateInterviewQuestions, generateConceptExplanation, @@ -195,6 +197,21 @@ const server = app.listen(PORT, "0.0.0.0", () => { } }); +// Attach WebSocket Server +const wss = new WebSocket.Server({ noServer: true }); +server.on("upgrade", (request, socket, head) => { + if (request.url === "/api/audio-stream") { + wss.handleUpgrade(request, socket, head, (ws) => { + wss.emit("connection", ws, request); + }); + } else { + socket.destroy(); + } +}); + +// Initialize the audio pipeline handler +setupAudioPipeline(wss); + server.on("error", (err) => { if (err.code === "EADDRINUSE") { console.error( diff --git a/frontend/src/hooks/useAudioPipeline.js b/frontend/src/hooks/useAudioPipeline.js new file mode 100644 index 00000000..beb93c9c --- /dev/null +++ b/frontend/src/hooks/useAudioPipeline.js @@ -0,0 +1,80 @@ +import { useState, useEffect, useRef } from 'react'; + +export const useAudioPipeline = (wsUrl) => { + const [isRecording, setIsRecording] = useState(false); + const [transcript, setTranscript] = useState(''); + const wsRef = useRef(null); + const mediaRecorderRef = useRef(null); + const audioContextRef = useRef(null); + + useEffect(() => { + // Initialize WebSocket connection to the backend + wsRef.current = new WebSocket(wsUrl); + + wsRef.current.onopen = () => { + console.log('Connected to Audio Pipeline WebSockets'); + }; + + wsRef.current.onmessage = async (event) => { + // Differentiate between JSON (transcript text) and Blob (audio bytes) + if (typeof event.data === 'string') { + try { + const data = JSON.parse(event.data); + if (data.type === 'transcript') { + setTranscript(data.text); + } + } catch (e) { + console.error("Failed to parse JSON transcript", e); + } + } else if (event.data instanceof Blob) { + // We received binary audio bytes from the TTS API via backend + console.log('Received TTS audio chunk'); + await playAudioChunk(event.data); + } + }; + + return () => { + if (wsRef.current) wsRef.current.close(); + }; + }, [wsUrl]); + + const playAudioChunk = async (blob) => { + if (!audioContextRef.current) { + audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)(); + } + const arrayBuffer = await blob.arrayBuffer(); + // In a full implementation, you decode AudioData. + // For now, this is a stub. + console.log(`Stub: Playing audio chunk of size ${arrayBuffer.byteLength} bytes`); + }; + + const startRecording = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + mediaRecorderRef.current = new MediaRecorder(stream, { mimeType: 'audio/webm' }); + + mediaRecorderRef.current.ondataavailable = (event) => { + if (event.data.size > 0 && wsRef.current?.readyState === WebSocket.OPEN) { + // Stream raw audio chunks to backend + wsRef.current.send(event.data); + } + }; + + // Request chunks every 250ms for low latency + mediaRecorderRef.current.start(250); + setIsRecording(true); + } catch (err) { + console.error('Microphone permission denied', err); + } + }; + + const stopRecording = () => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === 'recording') { + mediaRecorderRef.current.stop(); + mediaRecorderRef.current.stream.getTracks().forEach(track => track.stop()); + } + setIsRecording(false); + }; + + return { isRecording, startRecording, stopRecording, transcript }; +};