From 0be9f66e26eb51301c6bbce9177f805a22c8708a Mon Sep 17 00:00:00 2001 From: vipergsm Date: Fri, 31 Jul 2026 11:17:30 +0200 Subject: [PATCH 1/2] audio: survive an input format mismatch, and ask for mic access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two failures that both surface as something unrelated to their cause. `installTap(format:)` was passed `input.outputFormat(forBus: 0)`, read before `engine.start()`. That value can disagree with the format the node ends up running at, and AVAudioEngine then raises an Objective-C exception ("Failed to create tap due to format mismatch") that Swift cannot catch — the whole process dies on the first key press. `format: nil` means "whatever the node is actually running at"; the converter is now built lazily from the first buffer we really see, and cached per input format so the realtime tap callback does not allocate per buffer. Separately, nothing ever requested microphone access. An unauthorized input node reports a 0 ch / 0 Hz format and `engine.start()` fails with a bare CoreAudio -10868 ("format not supported"), which tells the user nothing they can act on. `requestAccess()` asks up front and `start()` guards on it. --- Sources/parrot/Audio/AudioCapture.swift | 65 +++++++++++++++++++++---- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/Sources/parrot/Audio/AudioCapture.swift b/Sources/parrot/Audio/AudioCapture.swift index 5ef1d55b..d4f14304 100644 --- a/Sources/parrot/Audio/AudioCapture.swift +++ b/Sources/parrot/Audio/AudioCapture.swift @@ -8,6 +8,33 @@ final class AudioCapture { enum CaptureError: Error { case engineStartFailed(Error) case converterCreationFailed + case microphoneNotAuthorized + } + + /// Ask for microphone access and wait for the answer. + /// + /// Without this the first `engine.start()` fails with a bare CoreAudio + /// -10868 ("format not supported"): an unauthorized input node reports a + /// 0 ch / 0 Hz format, which is unrelated to anything the user can act on. + /// The prompt is a GUI dialog, so this only works when the process was + /// launched from a GUI session — over SSH it returns false. + @discardableResult + static func requestAccess() -> Bool { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: + return true + case .notDetermined: + let sem = DispatchSemaphore(value: 0) + var granted = false + AVCaptureDevice.requestAccess(for: .audio) { ok in + granted = ok + sem.signal() + } + sem.wait() + return granted + default: + return false + } } static let targetSampleRate: Double = 16_000 @@ -25,9 +52,11 @@ final class AudioCapture { /// Begin recording. Idempotent — calling while already recording is a no-op. func start() throws { guard !isRecording else { return } + guard AudioCapture.requestAccess() else { + throw CaptureError.microphoneNotAuthorized + } let input = engine.inputNode - let inputFormat = input.outputFormat(forBus: 0) let targetFormat = AVAudioFormat( commonFormat: .pcmFormatFloat32, @@ -36,18 +65,23 @@ final class AudioCapture { interleaved: false )! - guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { - throw CaptureError.converterCreationFailed - } - self.converter = converter - lock.lock() samples.removeAll(keepingCapacity: true) lock.unlock() - - // Tap with input format; convert inside the callback. - input.installTap(onBus: 0, bufferSize: 4096, format: inputFormat) { [weak self] buffer, _ in - self?.process(buffer: buffer, converter: converter, targetFormat: targetFormat) + converter = nil + + // format: nil means "whatever the node is actually running at". + // Passing `input.outputFormat(forBus: 0)` explicitly looks equivalent + // but is not: the value read before the engine starts can disagree + // with the hardware format the node ends up with, and AVAudioEngine + // then throws an uncatchable Objective-C exception ("Failed to create + // tap due to format mismatch") that takes the whole process down. + // The converter is built lazily from the first buffer we actually see. + input.installTap(onBus: 0, bufferSize: 4096, format: nil) { [weak self] buffer, _ in + guard let self else { return } + guard let converter = self.converterFor(inputFormat: buffer.format, target: targetFormat) + else { return } + self.process(buffer: buffer, converter: converter, targetFormat: targetFormat) } engine.prepare() @@ -76,6 +110,17 @@ final class AudioCapture { return captured } + /// Cache a converter per input format. The tap callback runs on a realtime + /// audio thread, so this must not allocate on every buffer. + private func converterFor(inputFormat: AVAudioFormat, target: AVAudioFormat) -> AVAudioConverter? { + lock.lock() + defer { lock.unlock() } + if let converter, converter.inputFormat == inputFormat { return converter } + let made = AVAudioConverter(from: inputFormat, to: target) + converter = made + return made + } + private func process( buffer: AVAudioPCMBuffer, converter: AVAudioConverter, From 07efa2a7029abc74060884bea5dea3df2ea7667d Mon Sep 17 00:00:00 2001 From: vipergsm Date: Fri, 31 Jul 2026 11:17:30 +0200 Subject: [PATCH 2/2] cli: add --hotkey and --language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --hotkey: fn is not reachable on every keyboard. Non-Apple keyboards (tested on a Logitech MX Keys S) handle fn in firmware, so maskSecondaryFn never reaches macOS and parrot appears to start fine and then do nothing. The named choices are side-specific — CGEventFlags carries device-dependent low bits, and since `flags.contains(mask)` requires all bits of the mask, a value like 0x100010 matches right command only and leaves ⌘C/⌘V alone. Keyboards and remappers vary enough that no fixed list covers everyone, so a raw flags value copied out of --debug-hotkey is also accepted. The doctor's 🌐-key check is skipped when the hotkey has been rebound off fn, and the menu bar now names the actual key instead of hardcoding "fn". --language: no DecodingOptions were passed at all, so Whisper guessed the language from the audio. That is reliable on long recordings and much less so on the two-second clips dictation produces — Serbian came back decoded as Spanish. Passing an ISO 639-1 code pins it; omitting the flag keeps the previous auto-detect behaviour. --- Sources/parrot/Doctor.swift | 12 +- Sources/parrot/Parrot.swift | 105 +++++++++++++++++- .../Transcription/WhisperKitTranscriber.swift | 18 ++- Sources/parrot/UI/MenuBarController.swift | 8 +- 4 files changed, 127 insertions(+), 16 deletions(-) diff --git a/Sources/parrot/Doctor.swift b/Sources/parrot/Doctor.swift index a1dc4d79..ee8b0e28 100644 --- a/Sources/parrot/Doctor.swift +++ b/Sources/parrot/Doctor.swift @@ -16,12 +16,12 @@ struct Check { } enum DoctorReport { - static func run() -> [Check] { - [ - checkMicrophone(), - checkAccessibility(), - checkFnKeyMapping(), - ] + /// `includeFnKey: false` when the hotkey has been rebound off Fn — the + /// 🌐-key setting is then irrelevant and shouldn't block startup. + static func run(includeFnKey: Bool = true) -> [Check] { + var checks = [checkMicrophone(), checkAccessibility()] + if includeFnKey { checks.append(checkFnKeyMapping()) } + return checks } static func checkMicrophone() -> Check { diff --git a/Sources/parrot/Parrot.swift b/Sources/parrot/Parrot.swift index 05a69ebe..83a5fffe 100644 --- a/Sources/parrot/Parrot.swift +++ b/Sources/parrot/Parrot.swift @@ -1,5 +1,6 @@ import AppKit import ArgumentParser +import CoreGraphics import Foundation import WhisperKit @@ -13,6 +14,65 @@ struct Parrot: ParsableCommand { ) } +/// Which modifier acts as the push-to-talk key. +/// +/// The left/right variants matter: CGEventFlags carries device-dependent low +/// bits that distinguish the two physical keys, and `flags.contains(mask)` +/// requires *all* bits of the mask, so a side-specific raw value matches only +/// that side. Plain `.maskControl` etc. would match either side. +struct Hotkey: ExpressibleByArgument, Decodable { + /// How the key is named in the UI ("hold X to dictate"). + let label: String + let mask: CGEventFlags + let isFn: Bool + + static let table: [(String, UInt64, String)] = [ + ("fn", CGEventFlags.maskSecondaryFn.rawValue, "fn"), + ("right-command", 0x10_0010, "right ⌘"), + ("left-command", 0x10_0008, "left ⌘"), + ("right-option", 0x8_0040, "right ⌥"), + ("left-option", 0x8_0020, "left ⌥"), + ("right-control", 0x4_2100, "right control"), + ("left-control", 0x4_0101, "left control"), + ("right-shift", 0x2_0004, "right shift"), + ("left-shift", 0x2_0002, "left shift"), + ] + + init?(argument: String) { + if let hit = Self.table.first(where: { $0.0 == argument }) { + self.init(label: hit.2, mask: CGEventFlags(rawValue: hit.1), isFn: hit.0 == "fn") + return + } + // Escape hatch: a raw flags value copied straight out of + // --debug-hotkey, e.g. "0x80140". Keyboards vary enough (and + // third-party remappers exist) that no fixed list covers everyone. + let hex = argument.hasPrefix("0x") ? String(argument.dropFirst(2)) : argument + if let value = UInt64(hex, radix: 16), value != 0 { + self.init(label: "0x" + hex, mask: CGEventFlags(rawValue: value), isFn: false) + return + } + return nil + } + + var defaultValueDescription: String { label } + + private init(label: String, mask: CGEventFlags, isFn: Bool) { + self.label = label + self.mask = mask + self.isFn = isFn + } + + init(from decoder: any Decoder) throws { + let raw = try decoder.singleValueContainer().decode(String.self) + guard let parsed = Hotkey(argument: raw) else { + throw DecodingError.dataCorrupted( + .init(codingPath: [], debugDescription: "unknown hotkey: \(raw)") + ) + } + self = parsed + } +} + struct Run: ParsableCommand { static let configuration = CommandConfiguration( commandName: "run", @@ -34,9 +94,35 @@ struct Run: ParsableCommand { @Option(name: .long, help: "Model id to use. Defaults to the recommended model.") var model: String? + @Option( + name: .long, + help: """ + ISO 639-1 language code to transcribe in, e.g. sr, hr, de. \ + Omit to let Whisper auto-detect — reliable on long audio, much \ + less so on the few-second clips dictation produces. + """ + ) + var language: String? + + @Option( + name: .long, + help: ArgumentHelp( + """ + Modifier to hold: fn, right-command, left-command, right-option, \ + left-option, right-control, left-control, right-shift, left-shift \ + — or a raw flags value such as 0x80140. Non-Apple keyboards often \ + handle fn in firmware so it never reaches macOS; run \ + --debug-hotkey, hold the key you want, and pass the flags value it \ + prints. + """, + valueName: "key" + ) + ) + var hotkey: Hotkey = Hotkey(argument: "fn")! + func run() throws { if !skipDoctor { - let checks = DoctorReport.run() + let checks = DoctorReport.run(includeFnKey: hotkey.isFn) if !DoctorReport.allOK(checks) { FileHandle.standardError.write(Data("startup checks failed:\n".utf8)) DoctorReport.print(checks) @@ -61,7 +147,7 @@ struct Run: ParsableCommand { chosenModel = m } - let transcriber = WhisperKitTranscriber(model: chosenModel) + let transcriber = WhisperKitTranscriber(model: chosenModel, language: language) let warmupSemaphore = DispatchSemaphore(value: 0) var warmupError: Error? Task.detached { @@ -78,17 +164,26 @@ struct Run: ParsableCommand { throw ExitCode(1) } + // Ask up front rather than on the first key press: the prompt is a GUI + // dialog and blocks, so triggering it mid-dictation swallows the take. + if !AudioCapture.requestAccess() { + FileHandle.standardError.write(Data( + "microphone access denied — grant it in System Settings → Privacy & Security → Microphone (for the app you launched parrot from), then quit and relaunch.\n".utf8 + )) + throw ExitCode(1) + } + let app = NSApplication.shared app.setActivationPolicy(.accessory) - let monitor = HotkeyMonitor(debug: debugHotkey) + let monitor = HotkeyMonitor(mask: hotkey.mask, debug: debugHotkey) let capture = AudioCapture() let dumpWav = self.dumpWav let overlay: RecordingOverlay? = noOverlay ? nil : MainActor.assumeIsolated { RecordingOverlay() } if let overlay { capture.onLevel = { level in overlay.pushLevel(level) } } - let menuBar = MainActor.assumeIsolated { MenuBarController(modelID: chosenModel.id) } + let menuBar = MainActor.assumeIsolated { MenuBarController(modelID: chosenModel.id, hotkeyLabel: hotkey.label) } do { try monitor.start { event in @@ -169,7 +264,7 @@ struct Run: ParsableCommand { sigint.resume() signal(SIGINT, SIG_IGN) - FileHandle.standardError.write(Data("listening on fn hold · model: \(chosenModel.id) · ^C to quit\n".utf8)) + FileHandle.standardError.write(Data("listening on \(hotkey.label) hold · model: \(chosenModel.id) · ^C to quit\n".utf8)) app.run() } } diff --git a/Sources/parrot/Transcription/WhisperKitTranscriber.swift b/Sources/parrot/Transcription/WhisperKitTranscriber.swift index 8003194b..3871a17b 100644 --- a/Sources/parrot/Transcription/WhisperKitTranscriber.swift +++ b/Sources/parrot/Transcription/WhisperKitTranscriber.swift @@ -5,10 +5,13 @@ actor WhisperKitTranscriber: Transcriber { let modelID: String private let model: TranscriptionModel private var pipeline: WhisperKit? + /// ISO 639-1 code, or nil to let Whisper guess. + private let language: String? - init(model: TranscriptionModel) { + init(model: TranscriptionModel, language: String? = nil) { self.modelID = model.id self.model = model + self.language = language } /// Loads the model into memory; downloads first if not already on disk. @@ -29,7 +32,18 @@ actor WhisperKitTranscriber: Transcriber { if pipeline == nil { try await warmUp() } guard let pipeline else { throw TranscriberError.notLoaded } - let results = try await pipeline.transcribe(audioArray: audio) + // Without an explicit language Whisper guesses from the first seconds + // of audio. On short dictation clips it guesses badly — Serbian comes + // back decoded as Spanish. Pin it when the user told us. + var options = DecodingOptions() + options.task = .transcribe + if let language { + options.language = language + options.detectLanguage = false + options.usePrefillPrompt = true + } + + let results = try await pipeline.transcribe(audioArray: audio, decodeOptions: options) let raw = results.map(\.text).joined(separator: " ") return Self.sanitize(raw) } diff --git a/Sources/parrot/UI/MenuBarController.swift b/Sources/parrot/UI/MenuBarController.swift index 366ab060..d4200ed4 100644 --- a/Sources/parrot/UI/MenuBarController.swift +++ b/Sources/parrot/UI/MenuBarController.swift @@ -9,15 +9,17 @@ final class MenuBarController { private let modelLabel: NSMenuItem private let stateLabel: NSMenuItem private let modelID: String + private let idleTitle: String - init(modelID: String) { + init(modelID: String, hotkeyLabel: String = "fn") { self.modelID = modelID + self.idleTitle = "idle · hold \(hotkeyLabel) to dictate" self.statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength) let menu = NSMenu() menu.autoenablesItems = false - stateLabel = NSMenuItem(title: "idle · hold fn to dictate", action: nil, keyEquivalent: "") + stateLabel = NSMenuItem(title: idleTitle, action: nil, keyEquivalent: "") stateLabel.isEnabled = false menu.addItem(stateLabel) @@ -40,7 +42,7 @@ final class MenuBarController { } func setRecording(_ recording: Bool) { - stateLabel.title = recording ? "● recording" : "idle · hold fn to dictate" + stateLabel.title = recording ? "● recording" : idleTitle } func setTranscribing() {