diff --git a/Sources/parrot/Audio/AudioCapture.swift b/Sources/parrot/Audio/AudioCapture.swift index 5ef1d55b..d4f14304 100644 --- a/Sources/parrot/Audio/AudioCapture.swift +++ b/Sources/parrot/Audio/AudioCapture.swift @@ -8,6 +8,33 @@ final class AudioCapture { enum CaptureError: Error { case engineStartFailed(Error) case converterCreationFailed + case microphoneNotAuthorized + } + + /// Ask for microphone access and wait for the answer. + /// + /// Without this the first `engine.start()` fails with a bare CoreAudio + /// -10868 ("format not supported"): an unauthorized input node reports a + /// 0 ch / 0 Hz format, which is unrelated to anything the user can act on. + /// The prompt is a GUI dialog, so this only works when the process was + /// launched from a GUI session — over SSH it returns false. + @discardableResult + static func requestAccess() -> Bool { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: + return true + case .notDetermined: + let sem = DispatchSemaphore(value: 0) + var granted = false + AVCaptureDevice.requestAccess(for: .audio) { ok in + granted = ok + sem.signal() + } + sem.wait() + return granted + default: + return false + } } static let targetSampleRate: Double = 16_000 @@ -25,9 +52,11 @@ final class AudioCapture { /// Begin recording. Idempotent — calling while already recording is a no-op. func start() throws { guard !isRecording else { return } + guard AudioCapture.requestAccess() else { + throw CaptureError.microphoneNotAuthorized + } let input = engine.inputNode - let inputFormat = input.outputFormat(forBus: 0) let targetFormat = AVAudioFormat( commonFormat: .pcmFormatFloat32, @@ -36,18 +65,23 @@ final class AudioCapture { interleaved: false )! - guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { - throw CaptureError.converterCreationFailed - } - self.converter = converter - lock.lock() samples.removeAll(keepingCapacity: true) lock.unlock() - - // Tap with input format; convert inside the callback. - input.installTap(onBus: 0, bufferSize: 4096, format: inputFormat) { [weak self] buffer, _ in - self?.process(buffer: buffer, converter: converter, targetFormat: targetFormat) + converter = nil + + // format: nil means "whatever the node is actually running at". + // Passing `input.outputFormat(forBus: 0)` explicitly looks equivalent + // but is not: the value read before the engine starts can disagree + // with the hardware format the node ends up with, and AVAudioEngine + // then throws an uncatchable Objective-C exception ("Failed to create + // tap due to format mismatch") that takes the whole process down. + // The converter is built lazily from the first buffer we actually see. + input.installTap(onBus: 0, bufferSize: 4096, format: nil) { [weak self] buffer, _ in + guard let self else { return } + guard let converter = self.converterFor(inputFormat: buffer.format, target: targetFormat) + else { return } + self.process(buffer: buffer, converter: converter, targetFormat: targetFormat) } engine.prepare() @@ -76,6 +110,17 @@ final class AudioCapture { return captured } + /// Cache a converter per input format. The tap callback runs on a realtime + /// audio thread, so this must not allocate on every buffer. + private func converterFor(inputFormat: AVAudioFormat, target: AVAudioFormat) -> AVAudioConverter? { + lock.lock() + defer { lock.unlock() } + if let converter, converter.inputFormat == inputFormat { return converter } + let made = AVAudioConverter(from: inputFormat, to: target) + converter = made + return made + } + private func process( buffer: AVAudioPCMBuffer, converter: AVAudioConverter, diff --git a/Sources/parrot/Doctor.swift b/Sources/parrot/Doctor.swift index a1dc4d79..ee8b0e28 100644 --- a/Sources/parrot/Doctor.swift +++ b/Sources/parrot/Doctor.swift @@ -16,12 +16,12 @@ struct Check { } enum DoctorReport { - static func run() -> [Check] { - [ - checkMicrophone(), - checkAccessibility(), - checkFnKeyMapping(), - ] + /// `includeFnKey: false` when the hotkey has been rebound off Fn — the + /// 🌐-key setting is then irrelevant and shouldn't block startup. + static func run(includeFnKey: Bool = true) -> [Check] { + var checks = [checkMicrophone(), checkAccessibility()] + if includeFnKey { checks.append(checkFnKeyMapping()) } + return checks } static func checkMicrophone() -> Check { diff --git a/Sources/parrot/Parrot.swift b/Sources/parrot/Parrot.swift index 05a69ebe..83a5fffe 100644 --- a/Sources/parrot/Parrot.swift +++ b/Sources/parrot/Parrot.swift @@ -1,5 +1,6 @@ import AppKit import ArgumentParser +import CoreGraphics import Foundation import WhisperKit @@ -13,6 +14,65 @@ struct Parrot: ParsableCommand { ) } +/// Which modifier acts as the push-to-talk key. +/// +/// The left/right variants matter: CGEventFlags carries device-dependent low +/// bits that distinguish the two physical keys, and `flags.contains(mask)` +/// requires *all* bits of the mask, so a side-specific raw value matches only +/// that side. Plain `.maskControl` etc. would match either side. +struct Hotkey: ExpressibleByArgument, Decodable { + /// How the key is named in the UI ("hold X to dictate"). + let label: String + let mask: CGEventFlags + let isFn: Bool + + static let table: [(String, UInt64, String)] = [ + ("fn", CGEventFlags.maskSecondaryFn.rawValue, "fn"), + ("right-command", 0x10_0010, "right ⌘"), + ("left-command", 0x10_0008, "left ⌘"), + ("right-option", 0x8_0040, "right ⌥"), + ("left-option", 0x8_0020, "left ⌥"), + ("right-control", 0x4_2100, "right control"), + ("left-control", 0x4_0101, "left control"), + ("right-shift", 0x2_0004, "right shift"), + ("left-shift", 0x2_0002, "left shift"), + ] + + init?(argument: String) { + if let hit = Self.table.first(where: { $0.0 == argument }) { + self.init(label: hit.2, mask: CGEventFlags(rawValue: hit.1), isFn: hit.0 == "fn") + return + } + // Escape hatch: a raw flags value copied straight out of + // --debug-hotkey, e.g. "0x80140". Keyboards vary enough (and + // third-party remappers exist) that no fixed list covers everyone. + let hex = argument.hasPrefix("0x") ? String(argument.dropFirst(2)) : argument + if let value = UInt64(hex, radix: 16), value != 0 { + self.init(label: "0x" + hex, mask: CGEventFlags(rawValue: value), isFn: false) + return + } + return nil + } + + var defaultValueDescription: String { label } + + private init(label: String, mask: CGEventFlags, isFn: Bool) { + self.label = label + self.mask = mask + self.isFn = isFn + } + + init(from decoder: any Decoder) throws { + let raw = try decoder.singleValueContainer().decode(String.self) + guard let parsed = Hotkey(argument: raw) else { + throw DecodingError.dataCorrupted( + .init(codingPath: [], debugDescription: "unknown hotkey: \(raw)") + ) + } + self = parsed + } +} + struct Run: ParsableCommand { static let configuration = CommandConfiguration( commandName: "run", @@ -34,9 +94,35 @@ struct Run: ParsableCommand { @Option(name: .long, help: "Model id to use. Defaults to the recommended model.") var model: String? + @Option( + name: .long, + help: """ + ISO 639-1 language code to transcribe in, e.g. sr, hr, de. \ + Omit to let Whisper auto-detect — reliable on long audio, much \ + less so on the few-second clips dictation produces. + """ + ) + var language: String? + + @Option( + name: .long, + help: ArgumentHelp( + """ + Modifier to hold: fn, right-command, left-command, right-option, \ + left-option, right-control, left-control, right-shift, left-shift \ + — or a raw flags value such as 0x80140. Non-Apple keyboards often \ + handle fn in firmware so it never reaches macOS; run \ + --debug-hotkey, hold the key you want, and pass the flags value it \ + prints. + """, + valueName: "key" + ) + ) + var hotkey: Hotkey = Hotkey(argument: "fn")! + func run() throws { if !skipDoctor { - let checks = DoctorReport.run() + let checks = DoctorReport.run(includeFnKey: hotkey.isFn) if !DoctorReport.allOK(checks) { FileHandle.standardError.write(Data("startup checks failed:\n".utf8)) DoctorReport.print(checks) @@ -61,7 +147,7 @@ struct Run: ParsableCommand { chosenModel = m } - let transcriber = WhisperKitTranscriber(model: chosenModel) + let transcriber = WhisperKitTranscriber(model: chosenModel, language: language) let warmupSemaphore = DispatchSemaphore(value: 0) var warmupError: Error? Task.detached { @@ -78,17 +164,26 @@ struct Run: ParsableCommand { throw ExitCode(1) } + // Ask up front rather than on the first key press: the prompt is a GUI + // dialog and blocks, so triggering it mid-dictation swallows the take. + if !AudioCapture.requestAccess() { + FileHandle.standardError.write(Data( + "microphone access denied — grant it in System Settings → Privacy & Security → Microphone (for the app you launched parrot from), then quit and relaunch.\n".utf8 + )) + throw ExitCode(1) + } + let app = NSApplication.shared app.setActivationPolicy(.accessory) - let monitor = HotkeyMonitor(debug: debugHotkey) + let monitor = HotkeyMonitor(mask: hotkey.mask, debug: debugHotkey) let capture = AudioCapture() let dumpWav = self.dumpWav let overlay: RecordingOverlay? = noOverlay ? nil : MainActor.assumeIsolated { RecordingOverlay() } if let overlay { capture.onLevel = { level in overlay.pushLevel(level) } } - let menuBar = MainActor.assumeIsolated { MenuBarController(modelID: chosenModel.id) } + let menuBar = MainActor.assumeIsolated { MenuBarController(modelID: chosenModel.id, hotkeyLabel: hotkey.label) } do { try monitor.start { event in @@ -169,7 +264,7 @@ struct Run: ParsableCommand { sigint.resume() signal(SIGINT, SIG_IGN) - FileHandle.standardError.write(Data("listening on fn hold · model: \(chosenModel.id) · ^C to quit\n".utf8)) + FileHandle.standardError.write(Data("listening on \(hotkey.label) hold · model: \(chosenModel.id) · ^C to quit\n".utf8)) app.run() } } diff --git a/Sources/parrot/Transcription/WhisperKitTranscriber.swift b/Sources/parrot/Transcription/WhisperKitTranscriber.swift index 8003194b..3871a17b 100644 --- a/Sources/parrot/Transcription/WhisperKitTranscriber.swift +++ b/Sources/parrot/Transcription/WhisperKitTranscriber.swift @@ -5,10 +5,13 @@ actor WhisperKitTranscriber: Transcriber { let modelID: String private let model: TranscriptionModel private var pipeline: WhisperKit? + /// ISO 639-1 code, or nil to let Whisper guess. + private let language: String? - init(model: TranscriptionModel) { + init(model: TranscriptionModel, language: String? = nil) { self.modelID = model.id self.model = model + self.language = language } /// Loads the model into memory; downloads first if not already on disk. @@ -29,7 +32,18 @@ actor WhisperKitTranscriber: Transcriber { if pipeline == nil { try await warmUp() } guard let pipeline else { throw TranscriberError.notLoaded } - let results = try await pipeline.transcribe(audioArray: audio) + // Without an explicit language Whisper guesses from the first seconds + // of audio. On short dictation clips it guesses badly — Serbian comes + // back decoded as Spanish. Pin it when the user told us. + var options = DecodingOptions() + options.task = .transcribe + if let language { + options.language = language + options.detectLanguage = false + options.usePrefillPrompt = true + } + + let results = try await pipeline.transcribe(audioArray: audio, decodeOptions: options) let raw = results.map(\.text).joined(separator: " ") return Self.sanitize(raw) } diff --git a/Sources/parrot/UI/MenuBarController.swift b/Sources/parrot/UI/MenuBarController.swift index 366ab060..d4200ed4 100644 --- a/Sources/parrot/UI/MenuBarController.swift +++ b/Sources/parrot/UI/MenuBarController.swift @@ -9,15 +9,17 @@ final class MenuBarController { private let modelLabel: NSMenuItem private let stateLabel: NSMenuItem private let modelID: String + private let idleTitle: String - init(modelID: String) { + init(modelID: String, hotkeyLabel: String = "fn") { self.modelID = modelID + self.idleTitle = "idle · hold \(hotkeyLabel) to dictate" self.statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength) let menu = NSMenu() menu.autoenablesItems = false - stateLabel = NSMenuItem(title: "idle · hold fn to dictate", action: nil, keyEquivalent: "") + stateLabel = NSMenuItem(title: idleTitle, action: nil, keyEquivalent: "") stateLabel.isEnabled = false menu.addItem(stateLabel) @@ -40,7 +42,7 @@ final class MenuBarController { } func setRecording(_ recording: Bool) { - stateLabel.title = recording ? "● recording" : "idle · hold fn to dictate" + stateLabel.title = recording ? "● recording" : idleTitle } func setTranscribing() {