diff --git a/Sources/VoiceMemo/AudioRecorder.swift b/Sources/VoiceMemo/AudioRecorder.swift index a1395d8..38f6a5a 100644 --- a/Sources/VoiceMemo/AudioRecorder.swift +++ b/Sources/VoiceMemo/AudioRecorder.swift @@ -2,9 +2,11 @@ import Foundation import ScreenCaptureKit import AVFoundation import AppKit +import CoreMedia +import AudioToolbox @available(macOS 13.0, *) -class AudioRecorder: NSObject, ObservableObject, SCStreamOutput, SCStreamDelegate, AVCaptureAudioDataOutputSampleBufferDelegate { +class AudioRecorder: NSObject, ObservableObject, SCStreamOutput, SCStreamDelegate { @Published var isRecording = false @Published var statusMessage = "Ready to record" @Published var availableApps: [SCRunningApplication] = [] @@ -36,11 +38,14 @@ class AudioRecorder: NSObject, ObservableObject, SCStreamOutput, SCStreamDelegat private var remoteURL: URL? // Microphone Audio (Local) - private var micSession: AVCaptureSession? + private var micEngine: AVAudioEngine? private var micAssetWriter: AVAssetWriter? private var micAssetWriterInput: AVAssetWriterInput? private let micQueue = DispatchQueue(label: "cn.mistbit.voicememo.mic") private var isFirstMicBuffer = true + private var micStartSampleTime: AVAudioFramePosition? + private var micFallbackSampleTime: AVAudioFramePosition = 0 + private var micSampleRate: Double? private var localURL: URL? init(settings: SettingsStore) { @@ -116,6 +121,9 @@ class AudioRecorder: NSObject, ObservableObject, SCStreamOutput, SCStreamDelegat private func beginRecordingSession(app: SCRunningApplication) { isFirstRemoteBuffer = true isFirstMicBuffer = true + micStartSampleTime = nil + micFallbackSampleTime = 0 + micSampleRate = nil self.recordingStartTime = Date() self.recordingDuration = 0 @@ -245,29 +253,85 @@ class AudioRecorder: NSObject, ObservableObject, SCStreamOutput, SCStreamDelegat } } - // MARK: - Microphone Audio (AVCapture) + // MARK: - Microphone Audio (AVAudioEngine) private func startMicrophoneCapture() { micQueue.async { - self.micSession = AVCaptureSession() - guard let session = self.micSession else { return } + self.micEngine = AVAudioEngine() + guard let engine = self.micEngine else { return } + let inputNode = engine.inputNode - session.beginConfiguration() - guard let micDevice = AVCaptureDevice.default(for: .audio), - let micInput = try? AVCaptureDeviceInput(device: micDevice), - session.canAddInput(micInput) else { - self.settings.log("Cannot add mic input") - return + // Enable Echo Cancellation (Voice Processing I/O) + // This tells the system to subtract speaker output from mic input + var aecEnabled = false + do { + try inputNode.setVoiceProcessingEnabled(true) + aecEnabled = true + } catch { + self.settings.log("Mic AEC enable failed: \(error.localizedDescription)") + } + + let format = inputNode.inputFormat(forBus: 0) + + inputNode.installTap(onBus: 0, bufferSize: 4096, format: format) { [weak self] buffer, time in + guard let self = self else { return } + // Dispatch to micQueue to serialize with writer access + self.micQueue.async { + self.processMicBuffer(buffer, time: time) + } } - session.addInput(micInput) - let output = AVCaptureAudioDataOutput() - if session.canAddOutput(output) { - session.addOutput(output) - output.setSampleBufferDelegate(self, queue: self.micQueue) + do { + try engine.start() + if aecEnabled { + self.settings.log("Mic engine started with AEC enabled") + } else { + self.settings.log("Mic engine started without AEC") + } + } catch { + self.settings.log("Mic engine error: \(error.localizedDescription)") + } + } + } + + private func processMicBuffer(_ buffer: AVAudioPCMBuffer, time: AVAudioTime) { + let sampleRate = buffer.format.sampleRate + let effectiveSampleRate: Double + if sampleRate > 0 { + effectiveSampleRate = sampleRate + } else { + effectiveSampleRate = micSampleRate ?? 48000 + } + if micSampleRate == nil { + micSampleRate = effectiveSampleRate + } + let timescale = Int32(max(1, Int(effectiveSampleRate.rounded()))) + + let frameLength = AVAudioFramePosition(buffer.frameLength) + let presentationTime: CMTime + if time.isSampleTimeValid && sampleRate > 0 { + if micStartSampleTime == nil { + micStartSampleTime = time.sampleTime + } + let base = micStartSampleTime ?? time.sampleTime + let offset = max(0, time.sampleTime - base) + presentationTime = CMTime(value: CMTimeValue(offset), timescale: timescale) + } else { + presentationTime = CMTime(value: CMTimeValue(micFallbackSampleTime), timescale: timescale) + } + micFallbackSampleTime += frameLength + + guard let sampleBuffer = buffer.toCMSampleBuffer(presentationTimeStamp: presentationTime) else { return } + + if isFirstMicBuffer { + setupMicWriter(for: sampleBuffer) + isFirstMicBuffer = false + } + + if let input = micAssetWriterInput, input.isReadyForMoreMediaData { + if !input.append(sampleBuffer) { + settings.log("Mic append error: \(String(describing: micAssetWriter?.error))") } - session.commitConfiguration() - session.startRunning() } } @@ -353,8 +417,11 @@ class AudioRecorder: NSObject, ObservableObject, SCStreamOutput, SCStreamDelegat } // 2. Stop Microphone - micSession?.stopRunning() - micSession = nil + if let engine = micEngine { + engine.stop() + engine.inputNode.removeTap(onBus: 0) + micEngine = nil + } if let writer = micAssetWriter { if writer.status == .writing { micAssetWriterInput?.markAsFinished() @@ -480,19 +547,82 @@ class AudioRecorder: NSObject, ObservableObject, SCStreamOutput, SCStreamDelegat func stream(_ stream: SCStream, didStopWithError error: Error) { settings.log("SCStream error: \(error.localizedDescription)") } - - // AVCaptureAudioDataOutputSampleBufferDelegate - func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { - guard CMSampleBufferDataIsReady(sampleBuffer) else { return } +} + +extension AVAudioPCMBuffer { + func toCMSampleBuffer(presentationTimeStamp: CMTime) -> CMSampleBuffer? { + let formatDescription = self.format.formatDescription - if isFirstMicBuffer { - setupMicWriter(for: sampleBuffer) - isFirstMicBuffer = false + // Create CMSampleBuffer from AudioBufferList + // presentationTimeStamp is provided by caller + let frameCount = CMItemCount(self.frameLength) + let bufferListPointer = self.audioBufferList + let bufferCount = Int(bufferListPointer.pointee.mNumberBuffers) + let totalSize = withUnsafePointer(to: bufferListPointer.pointee.mBuffers) { bufferPtr in + let audioBuffers = UnsafeBufferPointer(start: bufferPtr, count: bufferCount) + return audioBuffers.reduce(0) { $0 + Int($1.mDataByteSize) } } - if let input = micAssetWriterInput, input.isReadyForMoreMediaData { - if !input.append(sampleBuffer) { - settings.log("Mic append error: \(String(describing: micAssetWriter?.error))") + + var blockBuffer: CMBlockBuffer? + let blockStatus = CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, + memoryBlock: nil, + blockLength: totalSize, + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, + offsetToData: 0, + dataLength: totalSize, + flags: 0, + blockBufferOut: &blockBuffer + ) + guard blockStatus == kCMBlockBufferNoErr, let blockBuffer else { return nil } + + var offset = 0 + let copyStatus = withUnsafePointer(to: bufferListPointer.pointee.mBuffers) { bufferPtr -> OSStatus in + let audioBuffers = UnsafeBufferPointer(start: bufferPtr, count: bufferCount) + for audioBuffer in audioBuffers { + if let data = audioBuffer.mData { + let status = CMBlockBufferReplaceDataBytes( + with: data, + blockBuffer: blockBuffer, + offsetIntoDestination: offset, + dataLength: Int(audioBuffer.mDataByteSize) + ) + if status != kCMBlockBufferNoErr { + return status + } + } + offset += Int(audioBuffer.mDataByteSize) } + return kCMBlockBufferNoErr + } + if copyStatus != kCMBlockBufferNoErr { + return nil } + + var timingInfo = CMSampleTimingInfo( + duration: CMTime(value: 1, timescale: CMTimeScale(self.format.sampleRate)), + presentationTimeStamp: presentationTimeStamp, + decodeTimeStamp: .invalid + ) + + var sampleBuffer: CMSampleBuffer? + let sampleStatus = CMSampleBufferCreate( + allocator: kCFAllocatorDefault, + dataBuffer: blockBuffer, + dataReady: true, + makeDataReadyCallback: nil, + refcon: nil, + formatDescription: formatDescription, + sampleCount: frameCount, + sampleTimingEntryCount: 1, + sampleTimingArray: &timingInfo, + sampleSizeEntryCount: 0, + sampleSizeArray: nil, + sampleBufferOut: &sampleBuffer + ) + + guard sampleStatus == noErr else { return nil } + return sampleBuffer } }