feat(audio): implement foundational full-duplex WebSocket audio pipeline for STT/TTS (fixes #2278)

backend/controllers/audioPipelineController.js

@@ -0,0 +1,38 @@
+// audioPipelineController.js
+// Handles full-duplex WebSocket connections for STT/LLM/TTS streaming.
+
+const setupAudioPipeline = (wss) => {
+  wss.on('connection', (ws) => {
+    console.log('New Audio Pipeline WebSocket Connection Established');
+
+    // In a real implementation, we would pipe these chunks to a streaming STT API
+    // (e.g., Whisper, Deepgram, Google Speech-to-Text).
+    ws.on('message', (message) => {
+      // 1. Receive binary audio chunks from the frontend client.
+      // 2. STT Provider converts to text.
+
+      // MOCK: Echo back a mock transcript and simulated audio chunk
+      const mockTranscript = "That's a great question regarding system design.";
+
+      // Send the text transcript back to the frontend for UI display
+      ws.send(JSON.stringify({ type: 'transcript', text: mockTranscript }));
+
+      // 3. The transcript is passed to the LLM (e.g., OpenAI, Gemini) to generate an AI response.
+      // 4. The LLM response is passed to a TTS Provider (e.g., ElevenLabs) to generate audio.
+
+      // MOCK: Send simulated binary audio bytes back to the client
+      const mockAudioBytes = Buffer.from('mock_audio_bytes_representing_speech');
+      ws.send(mockAudioBytes);
+    });
+
+    ws.on('close', () => {
+      console.log('Audio Pipeline WebSocket Connection Closed');
+    });
+
+    ws.on('error', (error) => {
+      console.error('Audio Pipeline WebSocket Error:', error);
+    });
+  });
+};
+
+module.exports = { setupAudioPipeline };

backend/server.js

@@ -12,6 +12,8 @@ const googleCalendarRoutes = require("./routes/googleCalendarRoutes");
 const path = require("path");
 const connectDB = require("./config/db");
 const cookieParser = require("cookie-parser");
+const WebSocket = require("ws");
+const { setupAudioPipeline } = require("./controllers/audioPipelineController");
 const {
   generateInterviewQuestions,
   generateConceptExplanation,
@@ -195,6 +197,21 @@ const server = app.listen(PORT, "0.0.0.0", () => {
   }
 });
 
+// Attach WebSocket Server
+const wss = new WebSocket.Server({ noServer: true });
+server.on("upgrade", (request, socket, head) => {
+  if (request.url === "/api/audio-stream") {
+    wss.handleUpgrade(request, socket, head, (ws) => {
+      wss.emit("connection", ws, request);
+    });
+  } else {
+    socket.destroy();
+  }
+});
+
+// Initialize the audio pipeline handler
+setupAudioPipeline(wss);
+
 server.on("error", (err) => {
   if (err.code === "EADDRINUSE") {
     console.error(

frontend/src/hooks/useAudioPipeline.js

@@ -0,0 +1,80 @@
+import { useState, useEffect, useRef } from 'react';
+
+export const useAudioPipeline = (wsUrl) => {
+  const [isRecording, setIsRecording] = useState(false);
+  const [transcript, setTranscript] = useState('');
+  const wsRef = useRef(null);
+  const mediaRecorderRef = useRef(null);
+  const audioContextRef = useRef(null);
+
+  useEffect(() => {
+    // Initialize WebSocket connection to the backend
+    wsRef.current = new WebSocket(wsUrl);
+
+    wsRef.current.onopen = () => {
+      console.log('Connected to Audio Pipeline WebSockets');
+    };
+
+    wsRef.current.onmessage = async (event) => {
+      // Differentiate between JSON (transcript text) and Blob (audio bytes)
+      if (typeof event.data === 'string') {
+        try {
+          const data = JSON.parse(event.data);
+          if (data.type === 'transcript') {
+            setTranscript(data.text);
+          }
+        } catch (e) {
+          console.error("Failed to parse JSON transcript", e);
+        }
+      } else if (event.data instanceof Blob) {
+        // We received binary audio bytes from the TTS API via backend
+        console.log('Received TTS audio chunk');
+        await playAudioChunk(event.data);
+      }
+    };
+
+    return () => {
+      if (wsRef.current) wsRef.current.close();
+    };
+  }, [wsUrl]);
+
+  const playAudioChunk = async (blob) => {
+    if (!audioContextRef.current) {
+      audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)();
+    }
+    const arrayBuffer = await blob.arrayBuffer();
+    // In a full implementation, you decode AudioData.
+    // For now, this is a stub.
+    console.log(`Stub: Playing audio chunk of size ${arrayBuffer.byteLength} bytes`);
+  };
+
+  const startRecording = async () => {
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      mediaRecorderRef.current = new MediaRecorder(stream, { mimeType: 'audio/webm' });
+
+      mediaRecorderRef.current.ondataavailable = (event) => {
+        if (event.data.size > 0 && wsRef.current?.readyState === WebSocket.OPEN) {
+          // Stream raw audio chunks to backend
+          wsRef.current.send(event.data);
+        }
+      };
+
+      // Request chunks every 250ms for low latency
+      mediaRecorderRef.current.start(250);
+      setIsRecording(true);
+    } catch (err) {
+      console.error('Microphone permission denied', err);
+    }
+  };
+
+  const stopRecording = () => {
+    if (mediaRecorderRef.current && mediaRecorderRef.current.state === 'recording') {
+      mediaRecorderRef.current.stop();
+      mediaRecorderRef.current.stream.getTracks().forEach(track => track.stop());
+    }
+    setIsRecording(false);
+  };
+
+  return { isRecording, startRecording, stopRecording, transcript };
+};