From f38904c7a17b1ff8932abcb542115ce2b4397267 Mon Sep 17 00:00:00 2001 From: Pratik Shankar Jadhav <44173994+pratikjadhav2726@users.noreply.github.com> Date: Sun, 11 Jan 2026 02:44:54 -0800 Subject: [PATCH 01/13] feat: speech interview helper assistant --- electron/AnswerAssistant.ts | 175 ++++++++++ electron/ConversationManager.ts | 152 +++++++++ electron/TranscriptionHelper.ts | 131 ++++++++ electron/ipcHandlers.ts | 141 ++++++++ electron/main.ts | 25 +- electron/preload.ts | 56 +++- electron/shortcuts.ts | 33 ++ package.json | 4 +- src/_pages/Queue.tsx | 10 +- src/_pages/Solutions.tsx | 6 + .../Conversation/ConversationSection.tsx | 309 ++++++++++++++++++ src/types/electron.d.ts | 13 + src/utils/audioRecorder.ts | 99 ++++++ stealth-run.sh | 0 14 files changed, 1147 insertions(+), 7 deletions(-) create mode 100644 electron/AnswerAssistant.ts create mode 100644 electron/ConversationManager.ts create mode 100644 electron/TranscriptionHelper.ts create mode 100644 src/components/Conversation/ConversationSection.tsx create mode 100644 src/utils/audioRecorder.ts mode change 100644 => 100755 stealth-run.sh diff --git a/electron/AnswerAssistant.ts b/electron/AnswerAssistant.ts new file mode 100644 index 00000000..3664306a --- /dev/null +++ b/electron/AnswerAssistant.ts @@ -0,0 +1,175 @@ +/** + * AnswerAssistant - Generates AI-powered answer suggestions based on conversation context + * Follows Single Responsibility Principle - only handles answer suggestion generation + * Uses Dependency Inversion Principle - depends on IConversationManager interface + */ +import OpenAI from 'openai'; +import { configHelper } from './ConfigHelper'; +import { IConversationManager } from './ConversationManager'; + +export interface AnswerSuggestion { + suggestions: string[]; + reasoning: string; +} + +export interface IAnswerAssistant { + generateAnswerSuggestions( + currentQuestion: string, + conversationManager: IConversationManager, + screenshotContext?: string + ): Promise; +} + +export class AnswerAssistant implements IAnswerAssistant { + private openai: OpenAI | null = null; + private readonly defaultModel: string = 'gpt-4o-mini'; + + constructor() { + this.initializeOpenAI(); + } + + /** + * Initializes OpenAI client with API key from config + */ + private initializeOpenAI(): void { + const config = configHelper.loadConfig(); + if (config.apiKey && config.apiKey.trim().length > 0) { + this.openai = new OpenAI({ apiKey: config.apiKey }); + } + } + + /** + * Generates answer suggestions based on conversation context + * @param currentQuestion - The current interviewer question + * @param conversationManager - Conversation manager instance (dependency injection) + * @param screenshotContext - Optional screenshot context for coding interviews + * @returns Promise resolving to answer suggestions + * @throws Error if OpenAI client not initialized or request fails + */ + public async generateAnswerSuggestions( + currentQuestion: string, + conversationManager: IConversationManager, + screenshotContext?: string + ): Promise { + if (!this.openai) { + throw new Error('OpenAI client not initialized. Please set API key.'); + } + + if (!currentQuestion || currentQuestion.trim().length === 0) { + throw new Error('Current question cannot be empty'); + } + + const conversationHistory = conversationManager.getConversationHistory(); + const previousAnswers = conversationManager.getIntervieweeAnswers(); + + const contextPrompt = this.buildContextPrompt( + currentQuestion, + conversationHistory, + previousAnswers, + screenshotContext + ); + + try { + const response = await this.openai.chat.completions.create({ + model: this.defaultModel, + messages: [ + { + role: 'system', + content: 'You are a helpful interview assistant that provides contextual answer suggestions based on conversation history. Provide concise, actionable suggestions.' + }, + { + role: 'user', + content: contextPrompt + } + ], + temperature: 0.7, + max_tokens: 500, + }); + + const suggestionsText = response.choices[0]?.message?.content || ''; + const suggestions = this.parseSuggestions(suggestionsText); + + return { + suggestions: suggestions.length > 0 + ? suggestions + : ['Consider answering based on your experience and background.'], + reasoning: 'Based on conversation history and previous answers', + }; + } catch (error: any) { + console.error('Error generating suggestions:', error); + + // Provide specific error messages + if (error.status === 401) { + throw new Error('Invalid API key. Please check your OpenAI API key in settings.'); + } else if (error.status === 429) { + throw new Error('Rate limit exceeded. Please try again in a moment.'); + } + + throw new Error(`Failed to generate suggestions: ${error.message || 'Unknown error'}`); + } + } + + /** + * Builds the context prompt for the AI + */ + private buildContextPrompt( + currentQuestion: string, + conversationHistory: string, + previousAnswers: string[], + screenshotContext?: string + ): string { + let prompt = `You are an AI assistant helping someone during an interview. +The interviewer just asked: "${currentQuestion}" + +Previous conversation: +${conversationHistory || 'No previous conversation yet.'} + +Previous answers the interviewee has given: +${previousAnswers.length > 0 ? previousAnswers.join('\n\n') : 'No previous answers yet.'} + +Based on the current question and conversation history, provide 3-5 bullet point suggestions that: +1. Directly answer the current question +2. Reference and build upon previous answers for consistency +3. Maintain a coherent narrative +4. Are specific and actionable + +Format as simple bullet points, one per line starting with "-".`; + + if (screenshotContext) { + prompt += `\n\nAdditional context from code screenshot: ${screenshotContext}`; + } + + return prompt; + } + + /** + * Parses AI response into structured suggestions + */ + private parseSuggestions(suggestionsText: string): string[] { + return suggestionsText + .split('\n') + .map(line => line.trim()) + .filter(line => { + // Match bullet points, numbered lists, or lines starting with common prefixes + return line.startsWith('-') || + line.startsWith('•') || + line.match(/^\d+\./) || + (line.length > 0 && line.length < 200); // Reasonable length + }) + .map(line => { + // Remove bullet/number prefixes + return line + .replace(/^[-•]\s*/, '') + .replace(/^\d+\.\s*/, '') + .trim(); + }) + .filter(line => line.length > 0 && line.length < 200); // Filter out empty or too long + } + + /** + * Checks if OpenAI client is initialized + */ + public isInitialized(): boolean { + return this.openai !== null; + } +} diff --git a/electron/ConversationManager.ts b/electron/ConversationManager.ts new file mode 100644 index 00000000..4df0319c --- /dev/null +++ b/electron/ConversationManager.ts @@ -0,0 +1,152 @@ +/** + * ConversationManager - Manages conversation state and messages + * Follows Single Responsibility Principle - only handles conversation state + * Uses EventEmitter for loose coupling (Observer pattern) + */ +import { EventEmitter } from 'events'; + +export interface ConversationMessage { + id: string; + speaker: 'interviewer' | 'interviewee'; + text: string; + timestamp: number; + edited?: boolean; +} + +export interface IConversationManager { + addMessage(text: string, speaker?: 'interviewer' | 'interviewee'): ConversationMessage; + toggleSpeaker(): 'interviewer' | 'interviewee'; + getCurrentSpeaker(): 'interviewer' | 'interviewee'; + getMessages(): ConversationMessage[]; + getConversationHistory(): string; + getIntervieweeAnswers(): string[]; + updateMessage(messageId: string, newText: string): boolean; + clearConversation(): void; + setSpeaker(speaker: 'interviewer' | 'interviewee'): void; +} + +export class ConversationManager extends EventEmitter implements IConversationManager { + private messages: ConversationMessage[] = []; + private currentSpeaker: 'interviewer' | 'interviewee' = 'interviewee'; + + /** + * Adds a new message to the conversation + * @param text - Message text + * @param speaker - Optional speaker override, uses current speaker if not provided + * @returns The created message + */ + public addMessage( + text: string, + speaker?: 'interviewer' | 'interviewee' + ): ConversationMessage { + if (!text || text.trim().length === 0) { + throw new Error('Message text cannot be empty'); + } + + const message: ConversationMessage = { + id: this.generateMessageId(), + speaker: speaker || this.currentSpeaker, + text: text.trim(), + timestamp: Date.now(), + }; + + this.messages.push(message); + this.emit('message-added', message); + return message; + } + + /** + * Toggles between interviewer and interviewee speaker modes + * @returns The new speaker mode + */ + public toggleSpeaker(): 'interviewer' | 'interviewee' { + this.currentSpeaker = this.currentSpeaker === 'interviewer' + ? 'interviewee' + : 'interviewer'; + this.emit('speaker-changed', this.currentSpeaker); + return this.currentSpeaker; + } + + /** + * Sets the current speaker mode + * @param speaker - Speaker mode to set + */ + public setSpeaker(speaker: 'interviewer' | 'interviewee'): void { + if (this.currentSpeaker !== speaker) { + this.currentSpeaker = speaker; + this.emit('speaker-changed', this.currentSpeaker); + } + } + + /** + * Gets the current speaker mode + */ + public getCurrentSpeaker(): 'interviewer' | 'interviewee' { + return this.currentSpeaker; + } + + /** + * Gets all messages in the conversation + * @returns Copy of messages array (immutable) + */ + public getMessages(): ConversationMessage[] { + return [...this.messages]; + } + + /** + * Gets conversation history as formatted string + * @returns Formatted conversation history + */ + public getConversationHistory(): string { + return this.messages + .map(msg => `[${msg.speaker === 'interviewer' ? 'Interviewer' : 'You'}] ${msg.text}`) + .join('\n\n'); + } + + /** + * Gets all answers from the interviewee + * @returns Array of interviewee answer texts + */ + public getIntervieweeAnswers(): string[] { + return this.messages + .filter(msg => msg.speaker === 'interviewee') + .map(msg => msg.text); + } + + /** + * Updates an existing message + * @param messageId - ID of message to update + * @param newText - New text for the message + * @returns True if message was found and updated, false otherwise + */ + public updateMessage(messageId: string, newText: string): boolean { + if (!newText || newText.trim().length === 0) { + return false; + } + + const message = this.messages.find(m => m.id === messageId); + if (message) { + message.text = newText.trim(); + message.edited = true; + this.emit('message-updated', message); + return true; + } + return false; + } + + /** + * Clears all messages and resets to default speaker + */ + public clearConversation(): void { + this.messages = []; + this.currentSpeaker = 'interviewee'; + this.emit('conversation-cleared'); + } + + /** + * Generates a unique message ID + */ + private generateMessageId(): string { + return `msg-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; + } +} diff --git a/electron/TranscriptionHelper.ts b/electron/TranscriptionHelper.ts new file mode 100644 index 00000000..bf8bf23e --- /dev/null +++ b/electron/TranscriptionHelper.ts @@ -0,0 +1,131 @@ +/** + * TranscriptionHelper - Handles audio transcription using OpenAI Whisper API + * Follows Single Responsibility Principle - only handles transcription + */ +import OpenAI from 'openai'; +import fs from 'fs'; +import path from 'path'; +import { app } from 'electron'; +import { configHelper } from './ConfigHelper'; + +export interface TranscriptionResult { + text: string; + language?: string; +} + +export interface ITranscriptionHelper { + transcribeAudio(audioBuffer: Buffer, mimeType?: string): Promise; +} + +export class TranscriptionHelper implements ITranscriptionHelper { + private openai: OpenAI | null = null; + private readonly tempDir: string; + + constructor() { + this.tempDir = path.join(app.getPath('temp'), 'audio-transcriptions'); + this.ensureTempDirectory(); + this.initializeOpenAI(); + } + + /** + * Initializes OpenAI client with API key from config + */ + private initializeOpenAI(): void { + const config = configHelper.loadConfig(); + if (config.apiKey && config.apiKey.trim().length > 0) { + this.openai = new OpenAI({ apiKey: config.apiKey }); + } + } + + /** + * Ensures temp directory exists for audio files + */ + private ensureTempDirectory(): void { + if (!fs.existsSync(this.tempDir)) { + fs.mkdirSync(this.tempDir, { recursive: true }); + } + } + + /** + * Transcribes audio buffer using OpenAI Whisper API + * @param audioBuffer - Audio data as Buffer + * @param mimeType - MIME type of the audio (default: 'audio/webm') + * @returns Promise resolving to transcription result + * @throws Error if transcription fails or OpenAI client not initialized + */ + public async transcribeAudio( + audioBuffer: Buffer, + mimeType: string = 'audio/webm' + ): Promise { + if (!this.openai) { + throw new Error('OpenAI client not initialized. Please set API key.'); + } + + if (!audioBuffer || audioBuffer.length === 0) { + throw new Error('Audio buffer is empty'); + } + + const tempPath = path.join(this.tempDir, `audio-${Date.now()}-${Math.random().toString(36).substring(7)}.webm`); + + try { + // Write buffer to temp file + fs.writeFileSync(tempPath, audioBuffer); + + // Create read stream for OpenAI API + const file = fs.createReadStream(tempPath); + + // Transcribe using Whisper API + const transcription = await this.openai.audio.transcriptions.create({ + file: file, + model: 'whisper-1', + language: 'en', // Optional: can be auto-detected + response_format: 'verbose_json', + }); + + // Clean up temp file + this.cleanupTempFile(tempPath); + + return { + text: transcription.text, + language: transcription.language, + }; + } catch (error: any) { + // Clean up on error + this.cleanupTempFile(tempPath); + + console.error('Transcription error:', error); + + // Provide more specific error messages + if (error.status === 401) { + throw new Error('Invalid API key. Please check your OpenAI API key in settings.'); + } else if (error.status === 429) { + throw new Error('Rate limit exceeded. Please try again in a moment.'); + } else if (error.message?.includes('file')) { + throw new Error('Invalid audio file format. Please try recording again.'); + } + + throw new Error(`Transcription failed: ${error.message || 'Unknown error'}`); + } + } + + /** + * Safely removes temporary file + */ + private cleanupTempFile(filePath: string): void { + try { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + } catch (error) { + console.error('Error cleaning up temp file:', error); + // Don't throw - cleanup errors shouldn't break the flow + } + } + + /** + * Checks if OpenAI client is initialized + */ + public isInitialized(): boolean { + return this.openai !== null; + } +} diff --git a/electron/ipcHandlers.ts b/electron/ipcHandlers.ts index f05a9aee..841b64b3 100644 --- a/electron/ipcHandlers.ts +++ b/electron/ipcHandlers.ts @@ -348,4 +348,145 @@ export function initializeIpcHandlers(deps: IIpcHandlerDeps): void { return { success: false, error: "Failed to delete last screenshot" } } }) + + // ============================================ + // Conversation & Transcription Handlers + // ============================================ + + // Transcription handler - receives audio buffer from renderer + ipcMain.handle("transcribe-audio", async (_event, audioBuffer: ArrayBuffer, mimeType: string) => { + try { + if (!deps.transcriptionHelper) { + return { success: false, error: "Transcription helper not initialized" }; + } + + const buffer = Buffer.from(audioBuffer); + const result = await deps.transcriptionHelper.transcribeAudio(buffer, mimeType); + return { success: true, result }; + } catch (error: any) { + console.error("Transcription error:", error); + return { success: false, error: error.message || "Transcription failed" }; + } + }) + + // Conversation message handlers + ipcMain.handle("add-conversation-message", (_event, text: string, speaker?: string) => { + try { + if (!deps.conversationManager) { + return { success: false, error: "Conversation manager not initialized" }; + } + + const message = deps.conversationManager.addMessage(text, speaker as any); + return { success: true, message }; + } catch (error: any) { + console.error("Error adding message:", error); + return { success: false, error: error.message || "Failed to add message" }; + } + }) + + ipcMain.handle("toggle-speaker", () => { + try { + if (!deps.conversationManager) { + return { success: false, error: "Conversation manager not initialized" }; + } + + const speaker = deps.conversationManager.toggleSpeaker(); + return { success: true, speaker }; + } catch (error: any) { + console.error("Error toggling speaker:", error); + return { success: false, error: error.message || "Failed to toggle speaker" }; + } + }) + + ipcMain.handle("get-conversation", () => { + try { + if (!deps.conversationManager) { + return { success: false, error: "Conversation manager not initialized", messages: [] }; + } + + const messages = deps.conversationManager.getMessages(); + return { success: true, messages }; + } catch (error: any) { + console.error("Error getting conversation:", error); + return { success: false, error: error.message || "Failed to get conversation", messages: [] }; + } + }) + + ipcMain.handle("clear-conversation", () => { + try { + if (!deps.conversationManager) { + return { success: false, error: "Conversation manager not initialized" }; + } + + deps.conversationManager.clearConversation(); + return { success: true }; + } catch (error: any) { + console.error("Error clearing conversation:", error); + return { success: false, error: error.message || "Failed to clear conversation" }; + } + }) + + ipcMain.handle("update-conversation-message", (_event, messageId: string, newText: string) => { + try { + if (!deps.conversationManager) { + return { success: false, error: "Conversation manager not initialized" }; + } + + const success = deps.conversationManager.updateMessage(messageId, newText); + return { success }; + } catch (error: any) { + console.error("Error updating message:", error); + return { success: false, error: error.message || "Failed to update message" }; + } + }) + + // AI suggestion handler + ipcMain.handle("get-answer-suggestions", async (_event, question: string, screenshotContext?: string) => { + try { + if (!deps.answerAssistant || !deps.conversationManager) { + return { success: false, error: "Answer assistant or conversation manager not initialized" }; + } + + const suggestions = await deps.answerAssistant.generateAnswerSuggestions( + question, + deps.conversationManager, + screenshotContext + ); + return { success: true, suggestions }; + } catch (error: any) { + console.error("Error generating suggestions:", error); + return { success: false, error: error.message || "Failed to generate suggestions" }; + } + }) + + // Event listeners for conversation events + if (deps.conversationManager) { + deps.conversationManager.on('message-added', (message) => { + const mainWindow = deps.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('conversation-message-added', message); + } + }); + + deps.conversationManager.on('speaker-changed', (speaker) => { + const mainWindow = deps.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('speaker-changed', speaker); + } + }); + + deps.conversationManager.on('message-updated', (message) => { + const mainWindow = deps.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('conversation-message-updated', message); + } + }); + + deps.conversationManager.on('conversation-cleared', () => { + const mainWindow = deps.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('conversation-cleared'); + } + }); + } } diff --git a/electron/main.ts b/electron/main.ts index 0eae187a..752e5963 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -29,6 +29,9 @@ const state = { screenshotHelper: null as ScreenshotHelper | null, shortcutsHelper: null as ShortcutsHelper | null, processingHelper: null as ProcessingHelper | null, + transcriptionHelper: null as import('./TranscriptionHelper').TranscriptionHelper | null, + conversationManager: null as import('./ConversationManager').ConversationManager | null, + answerAssistant: null as import('./AnswerAssistant').AnswerAssistant | null, // View and state management view: "queue" as "queue" | "solutions" | "debug", @@ -107,10 +110,13 @@ export interface IIpcHandlerDeps { moveWindowRight: () => void moveWindowUp: () => void moveWindowDown: () => void + transcriptionHelper?: import('./TranscriptionHelper').TranscriptionHelper + conversationManager?: import('./ConversationManager').ConversationManager + answerAssistant?: import('./AnswerAssistant').AnswerAssistant } // Initialize helpers -function initializeHelpers() { +async function initializeHelpers() { state.screenshotHelper = new ScreenshotHelper(state.view) state.processingHelper = new ProcessingHelper({ getScreenshotHelper, @@ -129,6 +135,16 @@ function initializeHelpers() { getHasDebugged, PROCESSING_EVENTS: state.PROCESSING_EVENTS } as IProcessingHelperDeps) + + // Initialize conversation and transcription helpers + const { TranscriptionHelper } = await import('./TranscriptionHelper') + const { ConversationManager } = await import('./ConversationManager') + const { AnswerAssistant } = await import('./AnswerAssistant') + + state.transcriptionHelper = new TranscriptionHelper() + state.conversationManager = new ConversationManager() + state.answerAssistant = new AnswerAssistant() + state.shortcutsHelper = new ShortcutsHelper({ getMainWindow, takeScreenshot, @@ -530,7 +546,7 @@ async function initializeApp() { console.log("No API key found in configuration. User will need to set up.") } - initializeHelpers() + await initializeHelpers() initializeIpcHandlers({ getMainWindow, setWindowDimensions, @@ -557,7 +573,10 @@ async function initializeApp() { ) ), moveWindowUp: () => moveWindowVertical((y) => y - state.step), - moveWindowDown: () => moveWindowVertical((y) => y + state.step) + moveWindowDown: () => moveWindowVertical((y) => y + state.step), + transcriptionHelper: state.transcriptionHelper, + conversationManager: state.conversationManager, + answerAssistant: state.answerAssistant }) await createWindow() state.shortcutsHelper?.registerGlobalShortcuts() diff --git a/electron/preload.ts b/electron/preload.ts index 85f32156..36b7ca93 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -236,7 +236,61 @@ const electronAPI = { ipcRenderer.removeListener("delete-last-screenshot", subscription) } }, - deleteLastScreenshot: () => ipcRenderer.invoke("delete-last-screenshot") + deleteLastScreenshot: () => ipcRenderer.invoke("delete-last-screenshot"), + + // ============================================ + // Conversation & Transcription Methods + // ============================================ + + // Transcription + transcribeAudio: (audioBuffer: ArrayBuffer, mimeType: string) => + ipcRenderer.invoke("transcribe-audio", audioBuffer, mimeType), + + // Conversation + addConversationMessage: (text: string, speaker?: string) => + ipcRenderer.invoke("add-conversation-message", text, speaker), + toggleSpeaker: () => ipcRenderer.invoke("toggle-speaker"), + getConversation: () => ipcRenderer.invoke("get-conversation"), + clearConversation: () => ipcRenderer.invoke("clear-conversation"), + updateConversationMessage: (messageId: string, newText: string) => + ipcRenderer.invoke("update-conversation-message", messageId, newText), + + // AI suggestions + getAnswerSuggestions: (question: string, screenshotContext?: string) => + ipcRenderer.invoke("get-answer-suggestions", question, screenshotContext), + + // Event listeners + onConversationMessageAdded: (callback: (message: any) => void) => { + const subscription = (_: any, message: any) => callback(message) + ipcRenderer.on("conversation-message-added", subscription) + return () => { + ipcRenderer.removeListener("conversation-message-added", subscription) + } + }, + + onSpeakerChanged: (callback: (speaker: string) => void) => { + const subscription = (_: any, speaker: string) => callback(speaker) + ipcRenderer.on("speaker-changed", subscription) + return () => { + ipcRenderer.removeListener("speaker-changed", subscription) + } + }, + + onConversationMessageUpdated: (callback: (message: any) => void) => { + const subscription = (_: any, message: any) => callback(message) + ipcRenderer.on("conversation-message-updated", subscription) + return () => { + ipcRenderer.removeListener("conversation-message-updated", subscription) + } + }, + + onConversationCleared: (callback: () => void) => { + const subscription = () => callback() + ipcRenderer.on("conversation-cleared", subscription) + return () => { + ipcRenderer.removeListener("conversation-cleared", subscription) + } + } } // Before exposing the API diff --git a/electron/shortcuts.ts b/electron/shortcuts.ts index a6fa5ebb..e53dc34a 100644 --- a/electron/shortcuts.ts +++ b/electron/shortcuts.ts @@ -106,6 +106,39 @@ export class ShortcutsHelper { this.deps.toggleMainWindow() }) + // Recording toggle (Ctrl/Cmd+M) + globalShortcut.register("CommandOrControl+M", async () => { + const mainWindow = this.deps.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + console.log("Command/Ctrl + M pressed. Toggling recording."); + try { + await mainWindow.webContents.executeJavaScript(` + (async () => { + const event = new CustomEvent('toggle-recording'); + window.dispatchEvent(event); + })(); + `); + } catch (error) { + console.error("Error toggling recording:", error); + } + } + }); + + // Speaker toggle (Ctrl/Cmd+Shift+M) + globalShortcut.register("CommandOrControl+Shift+M", async () => { + const mainWindow = this.deps.getMainWindow(); + if (mainWindow && !mainWindow.isDestroyed()) { + console.log("Command/Ctrl + Shift + M pressed. Toggling speaker."); + try { + await mainWindow.webContents.executeJavaScript(` + window.electronAPI.toggleSpeaker(); + `); + } catch (error) { + console.error("Error toggling speaker:", error); + } + } + }); + globalShortcut.register("CommandOrControl+Q", () => { console.log("Command/Ctrl + Q pressed. Quitting application.") app.quit() diff --git a/package.json b/package.json index 1fffcfb5..fee68383 100644 --- a/package.json +++ b/package.json @@ -144,7 +144,6 @@ "lucide-react": "^0.460.0", "openai": "^4.28.4", "react": "^18.2.0", - "react-code-blocks": "^0.1.6", "react-dom": "^18.2.0", "react-router-dom": "^6.28.1", "react-syntax-highlighter": "^15.6.1", @@ -202,5 +201,8 @@ "last 1 firefox version", "last 1 safari version" ] + }, + "overrides": { + "prismjs": ">=1.30.0" } } diff --git a/src/_pages/Queue.tsx b/src/_pages/Queue.tsx index c9194d5e..9b40be6e 100644 --- a/src/_pages/Queue.tsx +++ b/src/_pages/Queue.tsx @@ -2,6 +2,7 @@ import React, { useState, useEffect, useRef } from "react" import { useQuery } from "@tanstack/react-query" import ScreenshotQueue from "../components/Queue/ScreenshotQueue" import QueueCommands from "../components/Queue/QueueCommands" +import { ConversationSection } from "../components/Conversation/ConversationSection" import { useToast } from "../contexts/toast" import { Screenshot } from "../types/screenshots" @@ -137,9 +138,14 @@ const Queue: React.FC = ({ }; return ( -
+
-
+
+ {/* Conversation Section - Works independently of screenshots */} +
+ +
+ = ({ setLanguage={setLanguage} /> + {/* Conversation Section */} +
+ +
+ {/* Main Content - Modified width constraints */}
diff --git a/src/components/Conversation/ConversationSection.tsx b/src/components/Conversation/ConversationSection.tsx new file mode 100644 index 00000000..b3e0a2d0 --- /dev/null +++ b/src/components/Conversation/ConversationSection.tsx @@ -0,0 +1,309 @@ +/** + * ConversationSection - UI component for conversation recording and AI suggestions + * Follows Single Responsibility Principle - only handles conversation UI + * Uses existing ContentSection pattern for consistency + */ +import React, { useState, useEffect, useRef } from 'react'; +import { AudioRecorder } from '../../utils/audioRecorder'; + +interface ConversationMessage { + id: string; + speaker: 'interviewer' | 'interviewee'; + text: string; + timestamp: number; + edited?: boolean; +} + +interface AISuggestion { + suggestions: string[]; + reasoning: string; +} + +// Reuse the same ContentSection style from Solutions.tsx for consistency +const ContentSection = ({ + title, + content, + isLoading +}: { + title: string; + content: React.ReactNode; + isLoading: boolean; +}) => ( +
+

+ {title} +

+ {isLoading ? ( +
+

+ Processing... +

+
+ ) : ( +
+ {content} +
+ )} +
+); + +export const ConversationSection: React.FC = () => { + const [messages, setMessages] = useState([]); + const [isRecording, setIsRecording] = useState(false); + const [currentSpeaker, setCurrentSpeaker] = useState<'interviewer' | 'interviewee'>('interviewee'); + const [aiSuggestions, setAiSuggestions] = useState(null); + const [isProcessing, setIsProcessing] = useState(false); + const [recordingDuration, setRecordingDuration] = useState(0); + const messagesEndRef = useRef(null); + const audioRecorderRef = useRef(null); + const durationIntervalRef = useRef(null); + + useEffect(() => { + loadConversation(); + + const unsubscribeMessageAdded = window.electronAPI.onConversationMessageAdded((message: ConversationMessage) => { + setMessages(prev => [...prev, message]); + scrollToBottom(); + }); + + const unsubscribeSpeakerChanged = window.electronAPI.onSpeakerChanged((speaker: string) => { + setCurrentSpeaker(speaker as 'interviewer' | 'interviewee'); + }); + + const unsubscribeMessageUpdated = window.electronAPI.onConversationMessageUpdated((message: ConversationMessage) => { + setMessages(prev => prev.map(msg => msg.id === message.id ? message : msg)); + }); + + const unsubscribeCleared = window.electronAPI.onConversationCleared(() => { + setMessages([]); + setAiSuggestions(null); + }); + + // Listen for keyboard shortcut to toggle recording + const handleToggleRecording = async () => { + const currentIsRecording = audioRecorderRef.current?.getIsRecording() || false; + if (currentIsRecording) { + await handleStopRecording(); + } else { + await handleStartRecording(); + } + }; + + window.addEventListener('toggle-recording', handleToggleRecording); + + return () => { + unsubscribeMessageAdded(); + unsubscribeSpeakerChanged(); + unsubscribeMessageUpdated(); + unsubscribeCleared(); + window.removeEventListener('toggle-recording', handleToggleRecording); + if (durationIntervalRef.current) { + clearInterval(durationIntervalRef.current); + } + }; + }, []); + + const scrollToBottom = () => { + messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }; + + const loadConversation = async () => { + try { + const result = await window.electronAPI.getConversation(); + if (result.success) { + setMessages(result.messages); + scrollToBottom(); + } + } catch (error) { + console.error('Failed to load conversation:', error); + } + }; + + const handleStartRecording = async () => { + try { + if (!audioRecorderRef.current) { + audioRecorderRef.current = new AudioRecorder(); + } + + await audioRecorderRef.current.startRecording(); + setIsRecording(true); + setRecordingDuration(0); + + // Start duration counter + durationIntervalRef.current = setInterval(() => { + setRecordingDuration(prev => prev + 1); + }, 1000); + } catch (error: any) { + console.error('Failed to start recording:', error); + alert(error.message || 'Failed to start recording. Please check microphone permissions.'); + } + }; + + const handleStopRecording = async () => { + if (!audioRecorderRef.current || !isRecording) return; + + setIsRecording(false); + if (durationIntervalRef.current) { + clearInterval(durationIntervalRef.current); + durationIntervalRef.current = null; + } + + setIsProcessing(true); + try { + const audioBlob = await audioRecorderRef.current.stopRecording(); + + // Convert blob to ArrayBuffer + const arrayBuffer = await audioBlob.arrayBuffer(); + + // Transcribe + const transcribeResult = await window.electronAPI.transcribeAudio(arrayBuffer, audioBlob.type); + + if (transcribeResult.success && transcribeResult.result) { + const text = transcribeResult.result.text; + + // Add message + await window.electronAPI.addConversationMessage(text, currentSpeaker); + + // If interviewer question, get AI suggestions + if (currentSpeaker === 'interviewer') { + await fetchAISuggestions(text); + } else { + // Clear suggestions when interviewee responds + setAiSuggestions(null); + } + } + } catch (error: any) { + console.error('Failed to process recording:', error); + alert(error.message || 'Failed to process recording'); + } finally { + setIsProcessing(false); + setRecordingDuration(0); + } + }; + + const fetchAISuggestions = async (question: string) => { + try { + const result = await window.electronAPI.getAnswerSuggestions(question); + if (result.success && result.suggestions) { + setAiSuggestions(result.suggestions); + } + } catch (error: any) { + console.error('Failed to get AI suggestions:', error); + // Don't show alert for suggestion errors - it's not critical + } + }; + + const handleToggleSpeaker = async () => { + try { + const result = await window.electronAPI.toggleSpeaker(); + if (result.success) { + setCurrentSpeaker(result.speaker); + setAiSuggestions(null); // Clear suggestions when switching speaker + } + } catch (error) { + console.error('Failed to toggle speaker:', error); + } + }; + + const formatTime = (timestamp: number) => { + return new Date(timestamp).toLocaleTimeString([], { + hour: '2-digit', + minute: '2-digit' + }); + }; + + const formatDuration = (seconds: number) => { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins}:${secs.toString().padStart(2, '0')}`; + }; + + return ( +
+ {/* Recording Controls */} +
+ + + + + {isProcessing && ( + Processing... + )} +
+ + {/* Messages */} + {messages.length > 0 && ( + + {messages.map((message) => ( +
+
+
+ {message.speaker === 'interviewer' ? '👤 Interviewer' : '🎤 You'} +
+
{message.text}
+
+ {formatTime(message.timestamp)} +
+
+
+ ))} +
+ } + isLoading={false} + /> + )} + + {/* AI Suggestions - styled like "My Thoughts" from Solutions */} + {aiSuggestions && ( + +
+ {aiSuggestions.suggestions.map((suggestion, index) => ( +
+
+
{suggestion}
+
+ ))} +
+
+ } + isLoading={false} + /> + )} + +
+
+ ); +}; diff --git a/src/types/electron.d.ts b/src/types/electron.d.ts index a467cae3..59b067b9 100644 --- a/src/types/electron.d.ts +++ b/src/types/electron.d.ts @@ -61,6 +61,19 @@ export interface ElectronAPI { openLink: (url: string) => void onApiKeyInvalid: (callback: () => void) => () => void removeListener: (eventName: string, callback: (...args: any[]) => void) => void + + // Conversation & Transcription methods + transcribeAudio: (audioBuffer: ArrayBuffer, mimeType: string) => Promise<{ success: boolean; result?: { text: string; language?: string }; error?: string }> + addConversationMessage: (text: string, speaker?: string) => Promise<{ success: boolean; message?: any; error?: string }> + toggleSpeaker: () => Promise<{ success: boolean; speaker?: string; error?: string }> + getConversation: () => Promise<{ success: boolean; messages?: any[]; error?: string }> + clearConversation: () => Promise<{ success: boolean; error?: string }> + updateConversationMessage: (messageId: string, newText: string) => Promise<{ success: boolean; error?: string }> + getAnswerSuggestions: (question: string, screenshotContext?: string) => Promise<{ success: boolean; suggestions?: { suggestions: string[]; reasoning: string }; error?: string }> + onConversationMessageAdded: (callback: (message: any) => void) => () => void + onSpeakerChanged: (callback: (speaker: string) => void) => () => void + onConversationMessageUpdated: (callback: (message: any) => void) => () => void + onConversationCleared: (callback: () => void) => () => void } declare global { diff --git a/src/utils/audioRecorder.ts b/src/utils/audioRecorder.ts new file mode 100644 index 00000000..060bffe9 --- /dev/null +++ b/src/utils/audioRecorder.ts @@ -0,0 +1,99 @@ +/** + * AudioRecorder - Handles audio recording using Web Audio API + * Follows Single Responsibility Principle - only handles audio recording + */ +export interface IAudioRecorder { + startRecording(): Promise; + stopRecording(): Promise; + getIsRecording(): boolean; +} + +export class AudioRecorder implements IAudioRecorder { + private mediaRecorder: MediaRecorder | null = null; + private audioChunks: Blob[] = []; + private stream: MediaStream | null = null; + private isRecording: boolean = false; + + /** + * Starts audio recording from the user's microphone + * @throws Error if microphone access fails + */ + async startRecording(): Promise { + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + sampleRate: 16000, // Whisper prefers 16kHz + channelCount: 1, // Mono + echoCancellation: true, + noiseSuppression: true, + } + }); + + this.stream = stream; + this.audioChunks = []; + + // Use WebM format (works everywhere) + const options = { mimeType: 'audio/webm;codecs=opus' }; + this.mediaRecorder = new MediaRecorder(stream, options); + + this.mediaRecorder.ondataavailable = (event) => { + if (event.data.size > 0) { + this.audioChunks.push(event.data); + } + }; + + this.mediaRecorder.start(1000); // Collect data every second + this.isRecording = true; + } catch (error) { + console.error('Error starting recording:', error); + throw new Error('Failed to access microphone. Please check permissions.'); + } + } + + /** + * Stops recording and returns the audio blob + * @returns Promise resolving to the recorded audio blob + * @throws Error if not currently recording + */ + async stopRecording(): Promise { + return new Promise((resolve, reject) => { + if (!this.mediaRecorder || !this.isRecording) { + reject(new Error('Not currently recording')); + return; + } + + this.mediaRecorder.onstop = () => { + const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' }); + this.cleanup(); + resolve(audioBlob); + }; + + this.mediaRecorder.onerror = (error) => { + this.cleanup(); + reject(new Error(`Recording error: ${error}`)); + }; + + this.mediaRecorder.stop(); + this.isRecording = false; + }); + } + + /** + * Gets the current recording state + */ + getIsRecording(): boolean { + return this.isRecording; + } + + /** + * Cleans up resources (stops tracks, clears state) + */ + private cleanup(): void { + if (this.stream) { + this.stream.getTracks().forEach(track => track.stop()); + this.stream = null; + } + this.mediaRecorder = null; + this.audioChunks = []; + } +} diff --git a/stealth-run.sh b/stealth-run.sh old mode 100644 new mode 100755 From ee3592753c7ba970d651581e864dc8a3b6e2aa0e Mon Sep 17 00:00:00 2001 From: Pratik Shankar Jadhav <44173994+pratikjadhav2726@users.noreply.github.com> Date: Sun, 11 Jan 2026 03:23:55 -0800 Subject: [PATCH 02/13] feat: added conversations view with screenshot support for coding and normal interview as well --- electron/ProcessingHelper.ts | 49 ++++++- electron/main.ts | 4 +- src/_pages/Queue.tsx | 2 +- src/_pages/Solutions.tsx | 2 +- .../Conversation/ConversationSection.tsx | 130 ++++++++++-------- src/components/Queue/QueueCommands.tsx | 68 +++++++++ src/components/Settings/SettingsDialog.tsx | 6 + src/components/Solutions/SolutionCommands.tsx | 68 +++++++++ 8 files changed, 266 insertions(+), 63 deletions(-) diff --git a/electron/ProcessingHelper.ts b/electron/ProcessingHelper.ts index 0dcd26f0..9f40916b 100644 --- a/electron/ProcessingHelper.ts +++ b/electron/ProcessingHelper.ts @@ -66,6 +66,22 @@ export class ProcessingHelper { this.initializeAIClient(); }); } + + /** + * Get conversation context for integration with screenshot processing + */ + private getConversationContext(): string | null { + try { + const conversationManager = this.deps.getConversationManager?.(); + if (conversationManager) { + const history = conversationManager.getConversationHistory(); + return history && history.trim().length > 0 ? history : null; + } + } catch (error) { + console.error('Error getting conversation context:', error); + } + return null; + } /** * Initialize or reinitialize the AI client with current config @@ -473,18 +489,29 @@ export class ProcessingHelper { } } + // Get conversation context if available + const conversationContext = this.getConversationContext(); + // Use OpenAI for processing + const systemPrompt = conversationContext + ? `You are a coding challenge interpreter. Analyze the screenshot of the coding problem and extract all relevant information. Consider the conversation context provided. Return the information in JSON format with these fields: problem_statement, constraints, example_input, example_output. Just return the structured JSON without any other text.` + : "You are a coding challenge interpreter. Analyze the screenshot of the coding problem and extract all relevant information. Return the information in JSON format with these fields: problem_statement, constraints, example_input, example_output. Just return the structured JSON without any other text."; + + const userPrompt = conversationContext + ? `Extract the coding problem details from these screenshots. Consider the following conversation context:\n\n${conversationContext}\n\nReturn in JSON format. Preferred coding language we gonna use for this problem is ${language}.` + : `Extract the coding problem details from these screenshots. Return in JSON format. Preferred coding language we gonna use for this problem is ${language}.`; + const messages = [ { role: "system" as const, - content: "You are a coding challenge interpreter. Analyze the screenshot of the coding problem and extract all relevant information. Return the information in JSON format with these fields: problem_statement, constraints, example_input, example_output. Just return the structured JSON without any other text." + content: systemPrompt }, { role: "user" as const, content: [ { type: "text" as const, - text: `Extract the coding problem details from these screenshots. Return in JSON format. Preferred coding language we gonna use for this problem is ${language}.` + text: userPrompt }, ...imageDataList.map(data => ({ type: "image_url" as const, @@ -525,13 +552,20 @@ export class ProcessingHelper { } try { + // Get conversation context if available + const conversationContext = this.getConversationContext(); + + const geminiPrompt = conversationContext + ? `You are a coding challenge interpreter. Analyze the screenshots of the coding problem and extract all relevant information. Consider the following conversation context:\n\n${conversationContext}\n\nReturn the information in JSON format with these fields: problem_statement, constraints, example_input, example_output. Just return the structured JSON without any other text. Preferred coding language we gonna use for this problem is ${language}.` + : `You are a coding challenge interpreter. Analyze the screenshots of the coding problem and extract all relevant information. Return the information in JSON format with these fields: problem_statement, constraints, example_input, example_output. Just return the structured JSON without any other text. Preferred coding language we gonna use for this problem is ${language}.`; + // Create Gemini message structure const geminiMessages: GeminiMessage[] = [ { role: "user", parts: [ { - text: `You are a coding challenge interpreter. Analyze the screenshots of the coding problem and extract all relevant information. Return the information in JSON format with these fields: problem_statement, constraints, example_input, example_output. Just return the structured JSON without any other text. Preferred coding language we gonna use for this problem is ${language}.` + text: geminiPrompt }, ...imageDataList.map(data => ({ inlineData: { @@ -583,13 +617,20 @@ export class ProcessingHelper { } try { + // Get conversation context if available + const conversationContext = this.getConversationContext(); + + const anthropicPrompt = conversationContext + ? `Extract the coding problem details from these screenshots. Consider the following conversation context:\n\n${conversationContext}\n\nReturn in JSON format with these fields: problem_statement, constraints, example_input, example_output. Preferred coding language is ${language}.` + : `Extract the coding problem details from these screenshots. Return in JSON format with these fields: problem_statement, constraints, example_input, example_output. Preferred coding language is ${language}.`; + const messages = [ { role: "user" as const, content: [ { type: "text" as const, - text: `Extract the coding problem details from these screenshots. Return in JSON format with these fields: problem_statement, constraints, example_input, example_output. Preferred coding language is ${language}.` + text: anthropicPrompt }, ...imageDataList.map(data => ({ type: "image" as const, diff --git a/electron/main.ts b/electron/main.ts index 752e5963..ea751ec1 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -73,6 +73,7 @@ export interface IProcessingHelperDeps { setHasDebugged: (value: boolean) => void getHasDebugged: () => boolean PROCESSING_EVENTS: typeof state.PROCESSING_EVENTS + getConversationManager?: () => import('./ConversationManager').ConversationManager | null } export interface IShortcutsHelperDeps { @@ -133,7 +134,8 @@ async function initializeHelpers() { deleteScreenshot, setHasDebugged, getHasDebugged, - PROCESSING_EVENTS: state.PROCESSING_EVENTS + PROCESSING_EVENTS: state.PROCESSING_EVENTS, + getConversationManager: () => state.conversationManager } as IProcessingHelperDeps) // Initialize conversation and transcription helpers diff --git a/src/_pages/Queue.tsx b/src/_pages/Queue.tsx index 9b40be6e..a384bb32 100644 --- a/src/_pages/Queue.tsx +++ b/src/_pages/Queue.tsx @@ -142,7 +142,7 @@ const Queue: React.FC = ({
{/* Conversation Section - Works independently of screenshots */} -
+
diff --git a/src/_pages/Solutions.tsx b/src/_pages/Solutions.tsx index 0bdcf7d5..79cacb2c 100644 --- a/src/_pages/Solutions.tsx +++ b/src/_pages/Solutions.tsx @@ -502,7 +502,7 @@ const Solutions: React.FC = ({ /> {/* Conversation Section */} -
+
diff --git a/src/components/Conversation/ConversationSection.tsx b/src/components/Conversation/ConversationSection.tsx index b3e0a2d0..40e71342 100644 --- a/src/components/Conversation/ConversationSection.tsx +++ b/src/components/Conversation/ConversationSection.tsx @@ -2,8 +2,10 @@ * ConversationSection - UI component for conversation recording and AI suggestions * Follows Single Responsibility Principle - only handles conversation UI * Uses existing ContentSection pattern for consistency + * Integrates with screenshot system for cohesive experience */ import React, { useState, useEffect, useRef } from 'react'; +import { useQueryClient } from '@tanstack/react-query'; import { AudioRecorder } from '../../utils/audioRecorder'; interface ConversationMessage { @@ -48,6 +50,7 @@ const ContentSection = ({ ); export const ConversationSection: React.FC = () => { + const queryClient = useQueryClient(); const [messages, setMessages] = useState([]); const [isRecording, setIsRecording] = useState(false); const [currentSpeaker, setCurrentSpeaker] = useState<'interviewer' | 'interviewee'>('interviewee'); @@ -167,10 +170,8 @@ export const ConversationSection: React.FC = () => { // If interviewer question, get AI suggestions if (currentSpeaker === 'interviewer') { await fetchAISuggestions(text); - } else { - // Clear suggestions when interviewee responds - setAiSuggestions(null); } + // Don't clear suggestions when interviewee responds - user needs to see them! } } catch (error: any) { console.error('Failed to process recording:', error); @@ -183,7 +184,15 @@ export const ConversationSection: React.FC = () => { const fetchAISuggestions = async (question: string) => { try { - const result = await window.electronAPI.getAnswerSuggestions(question); + // Get problem statement from query cache if available (from screenshots) + const problemStatement = queryClient.getQueryData(['problem_statement']) as any; + let screenshotContext: string | undefined; + + if (problemStatement?.problem_statement) { + screenshotContext = `Problem Statement: ${problemStatement.problem_statement}\nConstraints: ${problemStatement.constraints || 'N/A'}\nExample Input: ${problemStatement.example_input || 'N/A'}\nExample Output: ${problemStatement.example_output || 'N/A'}`; + } + + const result = await window.electronAPI.getAnswerSuggestions(question, screenshotContext); if (result.success && result.suggestions) { setAiSuggestions(result.suggestions); } @@ -198,7 +207,7 @@ export const ConversationSection: React.FC = () => { const result = await window.electronAPI.toggleSpeaker(); if (result.success) { setCurrentSpeaker(result.speaker); - setAiSuggestions(null); // Clear suggestions when switching speaker + // Don't clear suggestions - user needs to see them when preparing their answer! } } catch (error) { console.error('Failed to toggle speaker:', error); @@ -219,9 +228,9 @@ export const ConversationSection: React.FC = () => { }; return ( -
- {/* Recording Controls */} -
+
+ {/* Recording Controls - Always visible at top */} +
- {/* Messages */} - {messages.length > 0 && ( - - {messages.map((message) => ( -
+ {/* Scrollable Conversation Area - Takes remaining space above AI suggestions */} +
+ {messages.length > 0 && ( + + {messages.map((message) => (
-
- {message.speaker === 'interviewer' ? '👤 Interviewer' : '🎤 You'} -
-
{message.text}
-
- {formatTime(message.timestamp)} +
+
+ {message.speaker === 'interviewer' ? '👤 Interviewer' : '🎤 You'} +
+
{message.text}
+
+ {formatTime(message.timestamp)} +
-
- ))} -
- } - isLoading={false} - /> - )} + ))} +
+ } + isLoading={false} + /> + )} +
+
- {/* AI Suggestions - styled like "My Thoughts" from Solutions */} + {/* AI Suggestions - Fixed at bottom, always visible, never scrolls */} {aiSuggestions && ( - -
- {aiSuggestions.suggestions.map((suggestion, index) => ( -
-
-
{suggestion}
-
- ))} +
+ +
+ {aiSuggestions.suggestions.map((suggestion, index) => ( +
+
+
{suggestion}
+
+ ))} +
-
- } - isLoading={false} - /> + } + isLoading={false} + /> +
)} - -
); }; diff --git a/src/components/Queue/QueueCommands.tsx b/src/components/Queue/QueueCommands.tsx index 88d6c283..f33fe518 100644 --- a/src/components/Queue/QueueCommands.tsx +++ b/src/components/Queue/QueueCommands.tsx @@ -321,6 +321,74 @@ const QueueCommands: React.FC = ({

+ {/* Start/Stop Recording Command */} +
{ + try { + const event = new CustomEvent('toggle-recording'); + window.dispatchEvent(event); + } catch (error) { + console.error("Error toggling recording:", error) + showToast( + "Error", + "Failed to toggle recording", + "error" + ) + } + }} + > +
+ Start/Stop Recording +
+ + {COMMAND_KEY} + + + M + +
+
+

+ Record interview conversation for transcription. +

+
+ + {/* Toggle Speaker Mode Command */} +
{ + try { + await window.electronAPI.toggleSpeaker(); + } catch (error) { + console.error("Error toggling speaker:", error) + showToast( + "Error", + "Failed to toggle speaker mode", + "error" + ) + } + }} + > +
+ Toggle Speaker Mode +
+ + {COMMAND_KEY} + + + Shift + + + M + +
+
+

+ Switch between Interviewer and You mode. +

+
+ {/* Solve Command */}
Take Screenshot
Ctrl+H / Cmd+H
+
Start/Stop Recording
+
Ctrl+M / Cmd+M
+ +
Toggle Speaker Mode
+
Ctrl+Shift+M / Cmd+Shift+M
+
Process Screenshots
Ctrl+Enter / Cmd+Enter
diff --git a/src/components/Solutions/SolutionCommands.tsx b/src/components/Solutions/SolutionCommands.tsx index 49497299..4701c854 100644 --- a/src/components/Solutions/SolutionCommands.tsx +++ b/src/components/Solutions/SolutionCommands.tsx @@ -324,6 +324,74 @@ const SolutionCommands: React.FC = ({

+ {/* Start/Stop Recording Command */} +
{ + try { + const event = new CustomEvent('toggle-recording'); + window.dispatchEvent(event); + } catch (error) { + console.error("Error toggling recording:", error) + showToast( + "Error", + "Failed to toggle recording", + "error" + ) + } + }} + > +
+ Start/Stop Recording +
+ + {COMMAND_KEY} + + + M + +
+
+

+ Record interview conversation for transcription. +

+
+ + {/* Toggle Speaker Mode Command */} +
{ + try { + await window.electronAPI.toggleSpeaker(); + } catch (error) { + console.error("Error toggling speaker:", error) + showToast( + "Error", + "Failed to toggle speaker mode", + "error" + ) + } + }} + > +
+ Toggle Speaker Mode +
+ + {COMMAND_KEY} + + + Shift + + + M + +
+
+

+ Switch between Interviewer and You mode. +

+
+ {extraScreenshots.length > 0 && (
Date: Sun, 11 Jan 2026 03:27:27 -0800 Subject: [PATCH 03/13] fix: cmd +m for toggle recording for ease --- .../Conversation/ConversationSection.tsx | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/components/Conversation/ConversationSection.tsx b/src/components/Conversation/ConversationSection.tsx index 40e71342..351ae463 100644 --- a/src/components/Conversation/ConversationSection.tsx +++ b/src/components/Conversation/ConversationSection.tsx @@ -61,6 +61,13 @@ export const ConversationSection: React.FC = () => { const audioRecorderRef = useRef(null); const durationIntervalRef = useRef(null); + // Use ref to track recording state for event listener + const isRecordingRef = useRef(false); + + useEffect(() => { + isRecordingRef.current = isRecording; + }, [isRecording]); + useEffect(() => { loadConversation(); @@ -84,7 +91,8 @@ export const ConversationSection: React.FC = () => { // Listen for keyboard shortcut to toggle recording const handleToggleRecording = async () => { - const currentIsRecording = audioRecorderRef.current?.getIsRecording() || false; + // Check actual recording state using ref to get latest value + const currentIsRecording = isRecordingRef.current || (audioRecorderRef.current?.getIsRecording() || false); if (currentIsRecording) { await handleStopRecording(); } else { @@ -124,12 +132,19 @@ export const ConversationSection: React.FC = () => { const handleStartRecording = async () => { try { + // Check if already recording + if (audioRecorderRef.current?.getIsRecording()) { + console.log('Already recording'); + return; + } + if (!audioRecorderRef.current) { audioRecorderRef.current = new AudioRecorder(); } await audioRecorderRef.current.startRecording(); setIsRecording(true); + isRecordingRef.current = true; setRecordingDuration(0); // Start duration counter @@ -143,9 +158,15 @@ export const ConversationSection: React.FC = () => { }; const handleStopRecording = async () => { - if (!audioRecorderRef.current || !isRecording) return; + // Check recorder state directly instead of React state to avoid stale closures + if (!audioRecorderRef.current || !audioRecorderRef.current.getIsRecording()) { + console.log('Not recording, cannot stop'); + return; + } setIsRecording(false); + isRecordingRef.current = false; + if (durationIntervalRef.current) { clearInterval(durationIntervalRef.current); durationIntervalRef.current = null; From b7beff3f73563228ff680552b0655477ed280e59 Mon Sep 17 00:00:00 2001 From: Pratik Shankar Jadhav <44173994+pratikjadhav2726@users.noreply.github.com> Date: Sun, 11 Jan 2026 03:48:38 -0800 Subject: [PATCH 04/13] feat: add speech recognition model configuration and validation for OpenAI integration+ Reduced the conversation section height from 500px to 350px in both the Queue and Solutions views. --- electron/ConfigHelper.ts | 25 ++++++++++- electron/TranscriptionHelper.ts | 25 +++++++++-- electron/preload.ts | 2 +- src/_pages/Queue.tsx | 2 +- src/_pages/Solutions.tsx | 2 +- src/components/Settings/SettingsDialog.tsx | 48 ++++++++++++++++++++++ src/types/electron.d.ts | 4 +- 7 files changed, 99 insertions(+), 9 deletions(-) diff --git a/electron/ConfigHelper.ts b/electron/ConfigHelper.ts index 6d1d2dba..afde8103 100644 --- a/electron/ConfigHelper.ts +++ b/electron/ConfigHelper.ts @@ -11,6 +11,7 @@ interface Config { extractionModel: string; solutionModel: string; debuggingModel: string; + speechRecognitionModel: string; // Speech recognition model (Whisper for OpenAI) language: string; opacity: number; } @@ -23,6 +24,7 @@ export class ConfigHelper extends EventEmitter { extractionModel: "gemini-2.0-flash", // Default to Flash for faster responses solutionModel: "gemini-2.0-flash", debuggingModel: "gemini-2.0-flash", + speechRecognitionModel: "whisper-1", // Default to Whisper for OpenAI language: "python", opacity: 1.0 }; @@ -110,6 +112,15 @@ export class ConfigHelper extends EventEmitter { config.debuggingModel = this.sanitizeModelSelection(config.debuggingModel, config.apiProvider); } + // Ensure speechRecognitionModel is valid (only whisper-1 for OpenAI) + if (config.speechRecognitionModel && config.apiProvider === "openai") { + if (config.speechRecognitionModel !== "whisper-1") { + config.speechRecognitionModel = "whisper-1"; + } + } else if (!config.speechRecognitionModel) { + config.speechRecognitionModel = this.defaultConfig.speechRecognitionModel; + } + return { ...this.defaultConfig, ...config @@ -174,14 +185,25 @@ export class ConfigHelper extends EventEmitter { updates.extractionModel = "gpt-4o"; updates.solutionModel = "gpt-4o"; updates.debuggingModel = "gpt-4o"; + updates.speechRecognitionModel = "whisper-1"; } else if (updates.apiProvider === "anthropic") { updates.extractionModel = "claude-3-7-sonnet-20250219"; updates.solutionModel = "claude-3-7-sonnet-20250219"; updates.debuggingModel = "claude-3-7-sonnet-20250219"; + // Speech recognition not supported for Anthropic } else { updates.extractionModel = "gemini-2.0-flash"; updates.solutionModel = "gemini-2.0-flash"; updates.debuggingModel = "gemini-2.0-flash"; + // Speech recognition not supported for Gemini + } + } + + // Validate speech recognition model (only whisper-1 is supported, and only for OpenAI) + if (updates.speechRecognitionModel) { + if (provider === "openai" && updates.speechRecognitionModel !== "whisper-1") { + console.warn(`Invalid speech recognition model: ${updates.speechRecognitionModel}. Only whisper-1 is supported for OpenAI.`); + updates.speechRecognitionModel = "whisper-1"; } } @@ -203,7 +225,8 @@ export class ConfigHelper extends EventEmitter { // This prevents re-initializing the AI client when only opacity changes if (updates.apiKey !== undefined || updates.apiProvider !== undefined || updates.extractionModel !== undefined || updates.solutionModel !== undefined || - updates.debuggingModel !== undefined || updates.language !== undefined) { + updates.debuggingModel !== undefined || updates.speechRecognitionModel !== undefined || + updates.language !== undefined) { this.emit('config-updated', newConfig); } diff --git a/electron/TranscriptionHelper.ts b/electron/TranscriptionHelper.ts index bf8bf23e..63836a79 100644 --- a/electron/TranscriptionHelper.ts +++ b/electron/TranscriptionHelper.ts @@ -25,15 +25,24 @@ export class TranscriptionHelper implements ITranscriptionHelper { this.tempDir = path.join(app.getPath('temp'), 'audio-transcriptions'); this.ensureTempDirectory(); this.initializeOpenAI(); + + // Listen for config changes to re-initialize + configHelper.on('config-updated', () => { + this.initializeOpenAI(); + }); } /** * Initializes OpenAI client with API key from config + * Only initializes if provider is OpenAI (Whisper only works with OpenAI) */ private initializeOpenAI(): void { const config = configHelper.loadConfig(); - if (config.apiKey && config.apiKey.trim().length > 0) { + if (config.apiProvider === "openai" && config.apiKey && config.apiKey.trim().length > 0) { this.openai = new OpenAI({ apiKey: config.apiKey }); + } else if (config.apiProvider !== "openai") { + console.log("Speech recognition is only supported with OpenAI provider"); + this.openai = null; } } @@ -57,8 +66,14 @@ export class TranscriptionHelper implements ITranscriptionHelper { audioBuffer: Buffer, mimeType: string = 'audio/webm' ): Promise { + const config = configHelper.loadConfig(); + + if (config.apiProvider !== "openai") { + throw new Error('Speech recognition is only supported with OpenAI provider. Please switch to OpenAI in settings.'); + } + if (!this.openai) { - throw new Error('OpenAI client not initialized. Please set API key.'); + throw new Error('OpenAI client not initialized. Please set OpenAI API key in settings.'); } if (!audioBuffer || audioBuffer.length === 0) { @@ -74,10 +89,14 @@ export class TranscriptionHelper implements ITranscriptionHelper { // Create read stream for OpenAI API const file = fs.createReadStream(tempPath); + // Get speech recognition model from config + const config = configHelper.loadConfig(); + const speechModel = config.speechRecognitionModel || 'whisper-1'; + // Transcribe using Whisper API const transcription = await this.openai.audio.transcriptions.create({ file: file, - model: 'whisper-1', + model: speechModel, language: 'en', // Optional: can be auto-detected response_format: 'verbose_json', }); diff --git a/electron/preload.ts b/electron/preload.ts index 36b7ca93..aa8e92ae 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -205,7 +205,7 @@ const electronAPI = { // New methods for OpenAI API integration getConfig: () => ipcRenderer.invoke("get-config"), - updateConfig: (config: { apiKey?: string; model?: string; language?: string; opacity?: number }) => + updateConfig: (config: { apiKey?: string; model?: string; language?: string; opacity?: number; apiProvider?: string; extractionModel?: string; solutionModel?: string; debuggingModel?: string; speechRecognitionModel?: string }) => ipcRenderer.invoke("update-config", config), onShowSettings: (callback: () => void) => { const subscription = () => callback() diff --git a/src/_pages/Queue.tsx b/src/_pages/Queue.tsx index a384bb32..db9a3058 100644 --- a/src/_pages/Queue.tsx +++ b/src/_pages/Queue.tsx @@ -142,7 +142,7 @@ const Queue: React.FC = ({
{/* Conversation Section - Works independently of screenshots */} -
+
diff --git a/src/_pages/Solutions.tsx b/src/_pages/Solutions.tsx index 79cacb2c..296c54dd 100644 --- a/src/_pages/Solutions.tsx +++ b/src/_pages/Solutions.tsx @@ -502,7 +502,7 @@ const Solutions: React.FC = ({ /> {/* Conversation Section */} -
+
diff --git a/src/components/Settings/SettingsDialog.tsx b/src/components/Settings/SettingsDialog.tsx index 3460ca8b..a9ecb34d 100644 --- a/src/components/Settings/SettingsDialog.tsx +++ b/src/components/Settings/SettingsDialog.tsx @@ -184,6 +184,7 @@ export function SettingsDialog({ open: externalOpen, onOpenChange }: SettingsDia const [extractionModel, setExtractionModel] = useState("gpt-4o"); const [solutionModel, setSolutionModel] = useState("gpt-4o"); const [debuggingModel, setDebuggingModel] = useState("gpt-4o"); + const [speechRecognitionModel, setSpeechRecognitionModel] = useState("whisper-1"); const [isLoading, setIsLoading] = useState(false); const { showToast } = useToast(); @@ -213,6 +214,7 @@ export function SettingsDialog({ open: externalOpen, onOpenChange }: SettingsDia extractionModel?: string; solutionModel?: string; debuggingModel?: string; + speechRecognitionModel?: string; } window.electronAPI @@ -223,6 +225,7 @@ export function SettingsDialog({ open: externalOpen, onOpenChange }: SettingsDia setExtractionModel(config.extractionModel || "gpt-4o"); setSolutionModel(config.solutionModel || "gpt-4o"); setDebuggingModel(config.debuggingModel || "gpt-4o"); + setSpeechRecognitionModel(config.speechRecognitionModel || "whisper-1"); }) .catch((error: unknown) => { console.error("Failed to load config:", error); @@ -243,14 +246,17 @@ export function SettingsDialog({ open: externalOpen, onOpenChange }: SettingsDia setExtractionModel("gpt-4o"); setSolutionModel("gpt-4o"); setDebuggingModel("gpt-4o"); + setSpeechRecognitionModel("whisper-1"); } else if (provider === "gemini") { setExtractionModel("gemini-1.5-pro"); setSolutionModel("gemini-1.5-pro"); setDebuggingModel("gemini-1.5-pro"); + setSpeechRecognitionModel("whisper-1"); // Keep whisper-1 but will show as not supported } else if (provider === "anthropic") { setExtractionModel("claude-3-7-sonnet-20250219"); setSolutionModel("claude-3-7-sonnet-20250219"); setDebuggingModel("claude-3-7-sonnet-20250219"); + setSpeechRecognitionModel("whisper-1"); // Keep whisper-1 but will show as not supported } }; @@ -263,6 +269,7 @@ export function SettingsDialog({ open: externalOpen, onOpenChange }: SettingsDia extractionModel, solutionModel, debuggingModel, + speechRecognitionModel, }); if (result) { @@ -569,6 +576,47 @@ export function SettingsDialog({ open: externalOpen, onOpenChange }: SettingsDia ); })}
+ + {/* Speech Recognition Model Selection */} +
+ +

+ Model used for transcribing interview conversations +

+ + {apiProvider === "openai" ? ( +
+
setSpeechRecognitionModel("whisper-1")} + > +
+
+
+

Whisper-1

+

OpenAI's speech-to-text model

+
+
+
+
+ ) : ( +
+

+ Speech recognition is only supported with OpenAI. Please switch to OpenAI provider to use this feature. +

+
+ )} +
+ +
+
+ + {/* Toggle Speaker Mode */} +
+ + {currentSpeaker === 'interviewer' ? 'Interviewer' : 'You'} + +
+ + + +
+
+ + {/* Clear Conversation */} +
+ Clear +
+ + {/* Keyboard Shortcuts Tooltip Trigger */} +
+ Shortcuts + + {/* Tooltip Content */} + {isTooltipVisible && ( +
+ {/* Add transparent bridge */} +
+
+
+

+ Keyboard Shortcuts +

+
+ {/* Start/Stop Recording */} +
+
+ Start/Stop Recording +
+ + {COMMAND_KEY} + + + M + +
+
+

+ Record interview conversation for transcription. +

+
+ + {/* Toggle Speaker Mode */} +
+
+ Toggle Speaker Mode +
+ + {COMMAND_KEY} + + + Shift + + + M + +
+
+

+ Switch between Interviewer and You mode. +

+
+
+
+
+
+ )} +
+ + {isProcessing && ( + Processing... + )} +
+
+
+ ); +}; diff --git a/src/components/Conversation/ConversationSection.tsx b/src/components/Conversation/ConversationSection.tsx index 351ae463..aa7ff0a8 100644 --- a/src/components/Conversation/ConversationSection.tsx +++ b/src/components/Conversation/ConversationSection.tsx @@ -7,6 +7,7 @@ import React, { useState, useEffect, useRef } from 'react'; import { useQueryClient } from '@tanstack/react-query'; import { AudioRecorder } from '../../utils/audioRecorder'; +import { ConversationCommands } from './ConversationCommands'; interface ConversationMessage { id: string; @@ -57,13 +58,26 @@ export const ConversationSection: React.FC = () => { const [aiSuggestions, setAiSuggestions] = useState(null); const [isProcessing, setIsProcessing] = useState(false); const [recordingDuration, setRecordingDuration] = useState(0); + const [tooltipHeight, setTooltipHeight] = useState(0); const messagesEndRef = useRef(null); const audioRecorderRef = useRef(null); const durationIntervalRef = useRef(null); - + // Use ref to track recording state for event listener const isRecordingRef = useRef(false); + const handleTooltipVisibilityChange = (visible: boolean, height: number) => { + setTooltipHeight(height); + }; + + const handleClearConversation = async () => { + try { + await window.electronAPI.clearConversation(); + } catch (error) { + console.error('Failed to clear conversation:', error); + } + }; + useEffect(() => { isRecordingRef.current = isRecording; }, [isRecording]); @@ -250,38 +264,26 @@ export const ConversationSection: React.FC = () => { return (
- {/* Recording Controls - Always visible at top */} -
- - - - - {isProcessing && ( - Processing... - )} -
+ {/* Conversation Commands Bar - Matches QueueCommands/SolutionCommands style */} + {/* Scrollable Conversation Area - Takes remaining space above AI suggestions */}
From 1c2961e1defdc60c1a32dfb980187e5794bedb0e Mon Sep 17 00:00:00 2001 From: Pratik Shankar Jadhav <44173994+pratikjadhav2726@users.noreply.github.com> Date: Sun, 11 Jan 2026 04:10:57 -0800 Subject: [PATCH 07/13] refactor: optimize code splitting by lazy loading components and syntax highlighter, adjust build configurations for environment-specific settings --- package.json | 2 -- src/App.tsx | 46 ++++++++++++++++++++++++++++------------ src/_pages/Debug.tsx | 31 ++++++++++++++++++++------- src/_pages/Solutions.tsx | 33 +++++++++++++++++++++------- vite.config.ts | 20 +++++++++++++---- 5 files changed, 97 insertions(+), 35 deletions(-) diff --git a/package.json b/package.json index fee68383..b92bc9cf 100644 --- a/package.json +++ b/package.json @@ -124,8 +124,6 @@ "dependencies": { "@anthropic-ai/sdk": "^0.39.0", "@electron/notarize": "^2.3.0", - "@emotion/react": "^11.11.0", - "@emotion/styled": "^11.11.0", "@radix-ui/react-dialog": "^1.1.2", "@radix-ui/react-label": "^2.1.0", "@radix-ui/react-slot": "^1.1.0", diff --git a/src/App.tsx b/src/App.tsx index f2dd348d..eab3d738 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,4 +1,4 @@ -import SubscribedApp from "./_pages/SubscribedApp" +import { lazy, Suspense } from "react" import { UpdateNotification } from "./components/UpdateNotification" import { QueryClient, @@ -14,14 +14,21 @@ import { } from "./components/ui/toast" import { ToastContext } from "./contexts/toast" import { WelcomeScreen } from "./components/WelcomeScreen" -import { SettingsDialog } from "./components/Settings/SettingsDialog" + +// Lazy load heavy components for better code splitting +const SubscribedApp = lazy(() => import("./_pages/SubscribedApp")) +const SettingsDialog = lazy(() => + import("./components/Settings/SettingsDialog").then(module => ({ + default: module.SettingsDialog + })) +) // Create a React Query client const queryClient = new QueryClient({ defaultOptions: { queries: { staleTime: 0, - gcTime: Infinity, + gcTime: 5 * 60 * 1000, // 5 minutes - prevents memory leaks retry: 1, refetchOnWindowFocus: false }, @@ -243,11 +250,20 @@ function App() {
{isInitialized ? ( hasApiKey ? ( - + +
+
+

Loading...

+
+
+ }> + + ) : ( ) @@ -264,11 +280,15 @@ function App() {
- {/* Settings Dialog */} - + {/* Settings Dialog - Lazy loaded */} + {isSettingsOpen && ( + + + + )} + import("react-syntax-highlighter").then(module => ({ + default: module.Prism + })) +) import ScreenshotQueue from "../components/Queue/ScreenshotQueue" import SolutionCommands from "../components/Solutions/SolutionCommands" import { Screenshot } from "../types/screenshots" @@ -32,10 +36,20 @@ const CodeSection = ({
) : (
- Loading syntax highlighter...
}> + { + // Dynamically import style to reduce initial bundle size + // This will be code-split by Vite + try { + const styleModule = require("react-syntax-highlighter/dist/esm/styles/prism") + return styleModule.dracula || {} + } catch { + return {} + } + })()} customStyle={{ maxWidth: "100%", margin: 0, @@ -47,7 +61,8 @@ const CodeSection = ({ wrapLongLines={true} > {code as string} - + +
)}
diff --git a/src/_pages/Solutions.tsx b/src/_pages/Solutions.tsx index 296c54dd..29085136 100644 --- a/src/_pages/Solutions.tsx +++ b/src/_pages/Solutions.tsx @@ -1,8 +1,13 @@ // Solutions.tsx -import React, { useState, useEffect, useRef } from "react" +import React, { useState, useEffect, useRef, lazy, Suspense } from "react" import { useQuery, useQueryClient } from "@tanstack/react-query" -import { Prism as SyntaxHighlighter } from "react-syntax-highlighter" -import { dracula } from "react-syntax-highlighter/dist/esm/styles/prism" +// Dynamic import for syntax highlighter - loaded only when code is displayed +// This reduces initial bundle size significantly +const SyntaxHighlighter = lazy(() => + import("react-syntax-highlighter").then(module => ({ + default: module.Prism + })) +) import ScreenshotQueue from "../components/Queue/ScreenshotQueue" @@ -82,10 +87,21 @@ const SolutionSection = ({ > {copied ? "Copied!" : "Copy"} - Loading syntax highlighter...
}> + { + // Dynamically import style to reduce initial bundle size + // This will be code-split by Vite + try { + // Use dynamic import for better tree-shaking + const styleModule = require("react-syntax-highlighter/dist/esm/styles/prism") + return styleModule.dracula || {} + } catch { + return {} + } + })()} customStyle={{ maxWidth: "100%", margin: 0, @@ -97,7 +113,8 @@ const SolutionSection = ({ wrapLongLines={true} > {content as string} - + +
)}
diff --git a/vite.config.ts b/vite.config.ts index d7360772..e6ba2d1a 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -14,8 +14,8 @@ export default defineConfig({ vite: { build: { outDir: "dist-electron", - sourcemap: true, - minify: false, + sourcemap: process.env.NODE_ENV === "development", + minify: process.env.NODE_ENV === "production" ? "esbuild" : false, rollupOptions: { external: ["electron"] } @@ -28,7 +28,8 @@ export default defineConfig({ vite: { build: { outDir: "dist-electron", - sourcemap: true, + sourcemap: process.env.NODE_ENV === "development", + minify: process.env.NODE_ENV === "production" ? "esbuild" : false, rollupOptions: { external: ["electron"] } @@ -48,7 +49,18 @@ export default defineConfig({ build: { outDir: "dist", emptyOutDir: true, - sourcemap: true + sourcemap: process.env.NODE_ENV === "development", + minify: process.env.NODE_ENV === "production" ? "esbuild" : false, + rollupOptions: { + output: { + manualChunks: { + 'react-vendor': ['react', 'react-dom', 'react-router-dom'], + 'query-vendor': ['@tanstack/react-query'], + 'ui-vendor': ['@radix-ui/react-dialog', '@radix-ui/react-toast', '@radix-ui/react-label', '@radix-ui/react-slot'], + 'icons': ['lucide-react'] + } + } + } }, resolve: { alias: { From 63251578b95e889044ee352b6fa335f11d49ce50 Mon Sep 17 00:00:00 2001 From: Pratik Shankar Jadhav <44173994+pratikjadhav2726@users.noreply.github.com> Date: Tue, 20 Jan 2026 00:32:10 -0800 Subject: [PATCH 08/13] feat: implement candidate profile feature for personalized AI suggestions in interview context --- electron/AnswerAssistant.ts | 35 +++++++++-- electron/ConfigHelper.ts | 12 +++- electron/ipcHandlers.ts | 5 +- electron/preload.ts | 6 +- .../Conversation/ConversationSection.tsx | 6 +- .../Settings/CandidateProfileSection.tsx | 59 +++++++++++++++++++ src/components/Settings/SettingsDialog.tsx | 40 ++++++++++++- src/types/electron.d.ts | 6 +- 8 files changed, 152 insertions(+), 17 deletions(-) create mode 100644 src/components/Settings/CandidateProfileSection.tsx diff --git a/electron/AnswerAssistant.ts b/electron/AnswerAssistant.ts index 3664306a..a9df0754 100644 --- a/electron/AnswerAssistant.ts +++ b/electron/AnswerAssistant.ts @@ -4,7 +4,7 @@ * Uses Dependency Inversion Principle - depends on IConversationManager interface */ import OpenAI from 'openai'; -import { configHelper } from './ConfigHelper'; +import { configHelper, CandidateProfile } from './ConfigHelper'; import { IConversationManager } from './ConversationManager'; export interface AnswerSuggestion { @@ -49,7 +49,8 @@ export class AnswerAssistant implements IAnswerAssistant { public async generateAnswerSuggestions( currentQuestion: string, conversationManager: IConversationManager, - screenshotContext?: string + screenshotContext?: string, + candidateProfile?: CandidateProfile ): Promise { if (!this.openai) { throw new Error('OpenAI client not initialized. Please set API key.'); @@ -62,11 +63,15 @@ export class AnswerAssistant implements IAnswerAssistant { const conversationHistory = conversationManager.getConversationHistory(); const previousAnswers = conversationManager.getIntervieweeAnswers(); + // Get candidate profile from config if not provided + const profile = candidateProfile || configHelper.loadConfig().candidateProfile; + const contextPrompt = this.buildContextPrompt( currentQuestion, conversationHistory, previousAnswers, - screenshotContext + screenshotContext, + profile ); try { @@ -116,7 +121,8 @@ export class AnswerAssistant implements IAnswerAssistant { currentQuestion: string, conversationHistory: string, previousAnswers: string[], - screenshotContext?: string + screenshotContext?: string, + candidateProfile?: CandidateProfile ): string { let prompt = `You are an AI assistant helping someone during an interview. The interviewer just asked: "${currentQuestion}" @@ -126,8 +132,27 @@ ${conversationHistory || 'No previous conversation yet.'} Previous answers the interviewee has given: ${previousAnswers.length > 0 ? previousAnswers.join('\n\n') : 'No previous answers yet.'} +`; + + // Add candidate profile context if available + if (candidateProfile) { + const profileSections: string[] = []; + + if (candidateProfile.name) { + profileSections.push(`Name: ${candidateProfile.name}`); + } + + if (candidateProfile.resume) { + profileSections.push(`Resume: ${candidateProfile.resume}`); + } + + if (profileSections.length > 0) { + prompt += `\n\nCandidate Profile (use this to personalize suggestions): +${profileSections.join('\n')}`; + } + } -Based on the current question and conversation history, provide 3-5 bullet point suggestions that: + prompt += `\n\nBased on the current question, conversation history${candidateProfile ? ', and candidate profile' : ''}, provide 3-5 bullet point suggestions that: 1. Directly answer the current question 2. Reference and build upon previous answers for consistency 3. Maintain a coherent narrative diff --git a/electron/ConfigHelper.ts b/electron/ConfigHelper.ts index afde8103..a95c5f9d 100644 --- a/electron/ConfigHelper.ts +++ b/electron/ConfigHelper.ts @@ -5,6 +5,11 @@ import { app } from "electron" import { EventEmitter } from "events" import { OpenAI } from "openai" +export interface CandidateProfile { + name?: string; + resume?: string; // Full resume text +} + interface Config { apiKey: string; apiProvider: "openai" | "gemini" | "anthropic"; // Added provider selection @@ -14,6 +19,7 @@ interface Config { speechRecognitionModel: string; // Speech recognition model (Whisper for OpenAI) language: string; opacity: number; + candidateProfile?: CandidateProfile; // Candidate profile for personalized AI suggestions } export class ConfigHelper extends EventEmitter { @@ -26,7 +32,11 @@ export class ConfigHelper extends EventEmitter { debuggingModel: "gemini-2.0-flash", speechRecognitionModel: "whisper-1", // Default to Whisper for OpenAI language: "python", - opacity: 1.0 + opacity: 1.0, + candidateProfile: { + name: "", + resume: "" + } }; constructor() { diff --git a/electron/ipcHandlers.ts b/electron/ipcHandlers.ts index 841b64b3..b2ce8bd1 100644 --- a/electron/ipcHandlers.ts +++ b/electron/ipcHandlers.ts @@ -441,7 +441,7 @@ export function initializeIpcHandlers(deps: IIpcHandlerDeps): void { }) // AI suggestion handler - ipcMain.handle("get-answer-suggestions", async (_event, question: string, screenshotContext?: string) => { + ipcMain.handle("get-answer-suggestions", async (_event, question: string, screenshotContext?: string, candidateProfile?: any) => { try { if (!deps.answerAssistant || !deps.conversationManager) { return { success: false, error: "Answer assistant or conversation manager not initialized" }; @@ -450,7 +450,8 @@ export function initializeIpcHandlers(deps: IIpcHandlerDeps): void { const suggestions = await deps.answerAssistant.generateAnswerSuggestions( question, deps.conversationManager, - screenshotContext + screenshotContext, + candidateProfile ); return { success: true, suggestions }; } catch (error: any) { diff --git a/electron/preload.ts b/electron/preload.ts index aa8e92ae..0b83247b 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -205,7 +205,7 @@ const electronAPI = { // New methods for OpenAI API integration getConfig: () => ipcRenderer.invoke("get-config"), - updateConfig: (config: { apiKey?: string; model?: string; language?: string; opacity?: number; apiProvider?: string; extractionModel?: string; solutionModel?: string; debuggingModel?: string; speechRecognitionModel?: string }) => + updateConfig: (config: { apiKey?: string; model?: string; language?: string; opacity?: number; apiProvider?: string; extractionModel?: string; solutionModel?: string; debuggingModel?: string; speechRecognitionModel?: string; candidateProfile?: any }) => ipcRenderer.invoke("update-config", config), onShowSettings: (callback: () => void) => { const subscription = () => callback() @@ -256,8 +256,8 @@ const electronAPI = { ipcRenderer.invoke("update-conversation-message", messageId, newText), // AI suggestions - getAnswerSuggestions: (question: string, screenshotContext?: string) => - ipcRenderer.invoke("get-answer-suggestions", question, screenshotContext), + getAnswerSuggestions: (question: string, screenshotContext?: string, candidateProfile?: any) => + ipcRenderer.invoke("get-answer-suggestions", question, screenshotContext, candidateProfile), // Event listeners onConversationMessageAdded: (callback: (message: any) => void) => { diff --git a/src/components/Conversation/ConversationSection.tsx b/src/components/Conversation/ConversationSection.tsx index aa7ff0a8..9a167589 100644 --- a/src/components/Conversation/ConversationSection.tsx +++ b/src/components/Conversation/ConversationSection.tsx @@ -227,7 +227,11 @@ export const ConversationSection: React.FC = () => { screenshotContext = `Problem Statement: ${problemStatement.problem_statement}\nConstraints: ${problemStatement.constraints || 'N/A'}\nExample Input: ${problemStatement.example_input || 'N/A'}\nExample Output: ${problemStatement.example_output || 'N/A'}`; } - const result = await window.electronAPI.getAnswerSuggestions(question, screenshotContext); + // Get candidate profile from config + const config = await window.electronAPI.getConfig(); + const candidateProfile = (config as any).candidateProfile; + + const result = await window.electronAPI.getAnswerSuggestions(question, screenshotContext, candidateProfile); if (result.success && result.suggestions) { setAiSuggestions(result.suggestions); } diff --git a/src/components/Settings/CandidateProfileSection.tsx b/src/components/Settings/CandidateProfileSection.tsx new file mode 100644 index 00000000..4c2ee372 --- /dev/null +++ b/src/components/Settings/CandidateProfileSection.tsx @@ -0,0 +1,59 @@ +/** + * CandidateProfileSection - Component for editing candidate profile + * Used in SettingsDialog to allow users to input their resume and details + */ +import React, { useState } from 'react'; +import { Input } from '../ui/input'; +import { Button } from '../ui/button'; + +export interface CandidateProfile { + name?: string; + resume?: string; +} + +interface CandidateProfileSectionProps { + profile: CandidateProfile; + onProfileChange: (profile: CandidateProfile) => void; +} + +export const CandidateProfileSection: React.FC = ({ + profile, + onProfileChange, +}) => { + const [localProfile, setLocalProfile] = useState(profile); + + const handleFieldChange = (field: keyof CandidateProfile, value: string) => { + const updated = { ...localProfile, [field]: value }; + setLocalProfile(updated); + onProfileChange(updated); + }; + + return ( +
+
+ + handleFieldChange('name', e.target.value)} + placeholder="Your name" + className="bg-black/30 border-white/10 text-white placeholder:text-white/40" + /> +
+ +
+ +