diff --git a/backend/controllers/audioPipelineController.js b/backend/controllers/audioPipelineController.js new file mode 100644 index 00000000..0006ff9f --- /dev/null +++ b/backend/controllers/audioPipelineController.js @@ -0,0 +1,38 @@ +// audioPipelineController.js +// Handles full-duplex WebSocket connections for STT/LLM/TTS streaming. + +const setupAudioPipeline = (wss) => { + wss.on('connection', (ws) => { + console.log('New Audio Pipeline WebSocket Connection Established'); + + // In a real implementation, we would pipe these chunks to a streaming STT API + // (e.g., Whisper, Deepgram, Google Speech-to-Text). + ws.on('message', (message) => { + // 1. Receive binary audio chunks from the frontend client. + // 2. STT Provider converts to text. + + // MOCK: Echo back a mock transcript and simulated audio chunk + const mockTranscript = "That's a great question regarding system design."; + + // Send the text transcript back to the frontend for UI display + ws.send(JSON.stringify({ type: 'transcript', text: mockTranscript })); + + // 3. The transcript is passed to the LLM (e.g., OpenAI, Gemini) to generate an AI response. + // 4. The LLM response is passed to a TTS Provider (e.g., ElevenLabs) to generate audio. + + // MOCK: Send simulated binary audio bytes back to the client + const mockAudioBytes = Buffer.from('mock_audio_bytes_representing_speech'); + ws.send(mockAudioBytes); + }); + + ws.on('close', () => { + console.log('Audio Pipeline WebSocket Connection Closed'); + }); + + ws.on('error', (error) => { + console.error('Audio Pipeline WebSocket Error:', error); + }); + }); +}; + +module.exports = { setupAudioPipeline }; diff --git a/backend/models/RepetitionState.js b/backend/models/RepetitionState.js new file mode 100644 index 00000000..ca1a4339 --- /dev/null +++ b/backend/models/RepetitionState.js @@ -0,0 +1,35 @@ +const mongoose = require("mongoose"); + +const repetitionStateSchema = new mongoose.Schema( + { + userId: { + type: mongoose.Schema.Types.ObjectId, + ref: "User", + required: true, + }, + flashcardId: { + type: mongoose.Schema.Types.ObjectId, + ref: "Flashcard", + required: true, + }, + easinessFactor: { + type: Number, + default: 2.5, + }, + interval: { + type: Number, + default: 0, + }, + repetitions: { + type: Number, + default: 0, + }, + nextReviewDate: { + type: Date, + default: Date.now, + }, + }, + { timestamps: true } +); + +module.exports = mongoose.model("RepetitionState", repetitionStateSchema); diff --git a/backend/server.js b/backend/server.js index 96db4300..5a78dc40 100644 --- a/backend/server.js +++ b/backend/server.js @@ -12,6 +12,8 @@ const googleCalendarRoutes = require("./routes/googleCalendarRoutes"); const path = require("path"); const connectDB = require("./config/db"); const cookieParser = require("cookie-parser"); +const WebSocket = require("ws"); +const { setupAudioPipeline } = require("./controllers/audioPipelineController"); const { generateInterviewQuestions, generateConceptExplanation, @@ -181,6 +183,25 @@ if (process.env.ADZUNA_APP_ID && process.env.ADZUNA_API_KEY) { setInterval(refreshJobCache, 24 * 60 * 60 * 1000); } +// Spaced Repetition CRON logic +const RepetitionState = require("./models/RepetitionState"); +const runSpacedRepetitionCron = async () => { + try { + const today = new Date(); + // This will aggregate and flag "due" cards for users. + // In a full implementation, you would update user queues here. + const dueCards = await RepetitionState.find({ nextReviewDate: { $lte: today } }); + if (dueCards.length > 0) { + console.log(`[CRON] Found ${dueCards.length} due flashcards for spaced repetition review.`); + } + } catch (error) { + console.error("[CRON] Spaced repetition error:", error); + } +}; +// Run the spaced repetition job every 24 hours +setInterval(runSpacedRepetitionCron, 24 * 60 * 60 * 1000); + + // Start Server const PORT = process.env.PORT || 5000; const server = app.listen(PORT, "0.0.0.0", () => { @@ -195,6 +216,21 @@ const server = app.listen(PORT, "0.0.0.0", () => { } }); +// Attach WebSocket Server +const wss = new WebSocket.Server({ noServer: true }); +server.on("upgrade", (request, socket, head) => { + if (request.url === "/api/audio-stream") { + wss.handleUpgrade(request, socket, head, (ws) => { + wss.emit("connection", ws, request); + }); + } else { + socket.destroy(); + } +}); + +// Initialize the audio pipeline handler +setupAudioPipeline(wss); + server.on("error", (err) => { if (err.code === "EADDRINUSE") { console.error( diff --git a/backend/utils/sm2.js b/backend/utils/sm2.js new file mode 100644 index 00000000..e7effdfb --- /dev/null +++ b/backend/utils/sm2.js @@ -0,0 +1,47 @@ +// SuperMemo-2 Spaced Repetition Algorithm + +/** + * Calculates the next state for a flashcard based on SM-2. + * + * @param {number} quality - User's grade (0-5). + * @param {number} repetitions - Current repetitions. + * @param {number} easinessFactor - Current easiness factor. + * @param {number} interval - Current interval (in days). + * @returns {Object} New state containing updated repetitions, easinessFactor, interval, and nextReviewDate. + */ +function calculateSM2(quality, repetitions, easinessFactor, interval) { + let newRepetitions = repetitions; + let newEasinessFactor = easinessFactor; + let newInterval = interval; + + if (quality >= 3) { + if (repetitions === 0) { + newInterval = 1; + } else if (repetitions === 1) { + newInterval = 6; + } else { + newInterval = Math.round(interval * easinessFactor); + } + newRepetitions += 1; + } else { + newRepetitions = 0; + newInterval = 1; + } + + // Update easiness factor + newEasinessFactor = easinessFactor + (0.1 - (5 - quality) * (0.08 + (5 - quality) * 0.02)); + if (newEasinessFactor < 1.3) newEasinessFactor = 1.3; + + // Calculate next review date + const nextReviewDate = new Date(); + nextReviewDate.setDate(nextReviewDate.getDate() + newInterval); + + return { + repetitions: newRepetitions, + easinessFactor: newEasinessFactor, + interval: newInterval, + nextReviewDate, + }; +} + +module.exports = { calculateSM2 }; diff --git a/frontend/src/hooks/useAudioPipeline.js b/frontend/src/hooks/useAudioPipeline.js new file mode 100644 index 00000000..beb93c9c --- /dev/null +++ b/frontend/src/hooks/useAudioPipeline.js @@ -0,0 +1,80 @@ +import { useState, useEffect, useRef } from 'react'; + +export const useAudioPipeline = (wsUrl) => { + const [isRecording, setIsRecording] = useState(false); + const [transcript, setTranscript] = useState(''); + const wsRef = useRef(null); + const mediaRecorderRef = useRef(null); + const audioContextRef = useRef(null); + + useEffect(() => { + // Initialize WebSocket connection to the backend + wsRef.current = new WebSocket(wsUrl); + + wsRef.current.onopen = () => { + console.log('Connected to Audio Pipeline WebSockets'); + }; + + wsRef.current.onmessage = async (event) => { + // Differentiate between JSON (transcript text) and Blob (audio bytes) + if (typeof event.data === 'string') { + try { + const data = JSON.parse(event.data); + if (data.type === 'transcript') { + setTranscript(data.text); + } + } catch (e) { + console.error("Failed to parse JSON transcript", e); + } + } else if (event.data instanceof Blob) { + // We received binary audio bytes from the TTS API via backend + console.log('Received TTS audio chunk'); + await playAudioChunk(event.data); + } + }; + + return () => { + if (wsRef.current) wsRef.current.close(); + }; + }, [wsUrl]); + + const playAudioChunk = async (blob) => { + if (!audioContextRef.current) { + audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)(); + } + const arrayBuffer = await blob.arrayBuffer(); + // In a full implementation, you decode AudioData. + // For now, this is a stub. + console.log(`Stub: Playing audio chunk of size ${arrayBuffer.byteLength} bytes`); + }; + + const startRecording = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + mediaRecorderRef.current = new MediaRecorder(stream, { mimeType: 'audio/webm' }); + + mediaRecorderRef.current.ondataavailable = (event) => { + if (event.data.size > 0 && wsRef.current?.readyState === WebSocket.OPEN) { + // Stream raw audio chunks to backend + wsRef.current.send(event.data); + } + }; + + // Request chunks every 250ms for low latency + mediaRecorderRef.current.start(250); + setIsRecording(true); + } catch (err) { + console.error('Microphone permission denied', err); + } + }; + + const stopRecording = () => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === 'recording') { + mediaRecorderRef.current.stop(); + mediaRecorderRef.current.stream.getTracks().forEach(track => track.stop()); + } + setIsRecording(false); + }; + + return { isRecording, startRecording, stopRecording, transcript }; +};