diff --git a/.gitignore b/.gitignore index 8a32409ce..709c219ca 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ src-tauri/resources/server-archive/* !src-tauri/resources/server-archive/placeholder.txt src-tauri/resources/node/* !src-tauri/resources/node/placeholder.txt +src-tauri/resources/whisper/* +!src-tauri/resources/whisper/placeholder.txt # Notarized release artifacts release/ diff --git a/scripts/release.sh b/scripts/release.sh index 0fce017f1..d81f0e6d0 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -367,6 +367,10 @@ echo " found: $APP_PATH" # release instead of shipping that. require_file "$APP_PATH/Contents/Resources/resources/node/bin/node" echo " bundled Node runtime present" +WHISPER_CLI="$APP_PATH/Contents/Resources/resources/whisper/whisper-cli" +require_file "$WHISPER_CLI" +"$WHISPER_CLI" --version >/dev/null +echo " bundled Whisper runtime present" echo "==> Signing every native binary inside the bundle" # Apple deprecated --deep; sign inner native binaries explicitly so each one diff --git a/scripts/run-tests.mjs b/scripts/run-tests.mjs index e2bd03f9b..ce97d607b 100644 --- a/scripts/run-tests.mjs +++ b/scripts/run-tests.mjs @@ -1085,6 +1085,7 @@ export const SUITES = { "src/app/api/profile-route.test.ts", "src/lib/voice/registry.test.ts", "src/lib/voice/speech-models.test.ts", + "src/lib/voice/sidecar-whisper.test.ts", "src/lib/voice/local-loop.test.ts", "src/lib/voice/speech-loop.test.ts", "src/lib/voice/native-stt.test.ts", @@ -1100,6 +1101,7 @@ export const SUITES = { "src/lib/voice/use-dictation.test.ts", "src/app/api/voice/session/route.test.ts", "src/app/api/voice/engines/route.test.ts", + "src/app/api/voice/engines/whisper/route.test.ts", "src/app/api/voice/preview/route.test.ts", "src/app/api/voice/local/chat/route.test.ts", "src/app/api/voice/elevenlabs/tts/route.test.ts", diff --git a/scripts/sidecar-bundle-deps.test.mjs b/scripts/sidecar-bundle-deps.test.mjs index d41e6f871..ed6183054 100644 --- a/scripts/sidecar-bundle-deps.test.mjs +++ b/scripts/sidecar-bundle-deps.test.mjs @@ -68,7 +68,7 @@ for (const forbiddenRoot of [ ]) { assert.match(closureSource, new RegExp(forbiddenRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")), `runtime verifier must exclude ${forbiddenRoot}`); } -assert.match(closureSource, /fileCount: 5_693/, "runtime closure must retain combined cross-platform headroom"); +assert.match(closureSource, /fileCount: 5_706/, "runtime closure must retain combined cross-platform headroom"); assert.match(closureSource, /unpackedBytes: 200 \* 1024 \* 1024 - 1/, "runtime closure must stay strictly below 200 MiB expanded"); // App-size: runtime bundles must drop test/dev packages and metadata that are @@ -135,13 +135,17 @@ assert.doesNotMatch( // Tauri assembles the app. assert.deepEqual( windowsConfig.bundle.resources, - ["resources/server-archive/**/*", "resources/node/**/*"], - "Windows resources must replace the expanded sidecar with its bounded archive", + ["resources/server-archive/**/*", "resources/node/**/*", "resources/whisper/**/*"], + "Windows resources must replace the expanded sidecar with its bounded archive while retaining bundled runtimes", ); assert.ok( baseConfig.bundle.resources.includes("resources/server/**/*"), "non-Windows bundles must retain the expanded tree for nested native signing", ); +assert.ok( + baseConfig.bundle.resources.includes("resources/whisper/**/*"), + "desktop bundles must retain the local Whisper runtime", +); assert.match(src, /WINDOWS_ARCHIVE/, "Windows sidecar must be emitted as a tar.zst archive"); assert.match(src, /sidecar-archive-manifest\.mjs/, "archive generation must emit its integrity and size manifest"); assert.match(src, /\.server\.tar\.zst\.\$\$\.tmp/, "archive generation must use a same-directory staging path"); @@ -171,7 +175,7 @@ assert.match( ); assert.match( rustArchiveSource, - /const MAX_FILE_COUNT: u64 = 5_693;/, + /const MAX_FILE_COUNT: u64 = 5_706;/, "Windows archive extractor must accept the shared runtime file-count budget", ); assert.match(manifestSource, /isSymbolicLink\(\)/, "archive input must reject symlinks"); diff --git a/scripts/sidecar-bundle.sh b/scripts/sidecar-bundle.sh index 863e8829c..1162faa13 100755 --- a/scripts/sidecar-bundle.sh +++ b/scripts/sidecar-bundle.sh @@ -275,6 +275,9 @@ copy_node_shared_runtime "$NODE_BIN" "$BUNDLED_NODE_DIR" "$BUNDLED_NODE_DIR/bin/$NODE_NAME" -e "process.exit(0)" >/dev/null printf "generated at release build time\n" > "$BUNDLED_NODE_DIR/placeholder.txt" +echo "==> staging bundled Whisper runtime" +bash "$ROOT/scripts/whisper-runtime-bundle.sh" + # Next.js + pnpm leaves a node_modules full of pnpm-style symlinks # (.pnpm/* paths) that don't survive the copy into the .app bundle. Recreate # production deps from the committed pnpm lockfile in a staging dir, then copy diff --git a/scripts/sidecar-runtime-closure.mjs b/scripts/sidecar-runtime-closure.mjs index 184492bf4..3b151c7d0 100644 --- a/scripts/sidecar-runtime-closure.mjs +++ b/scripts/sidecar-runtime-closure.mjs @@ -127,7 +127,10 @@ export const SIDECAR_RUNTIME_BUDGETS = Object.freeze({ // ([id]/files/[key]) traces two more chunks; the PR merge tree measured // 5,683 on Windows. Retain the ten-file buffer without relaxing the byte // ceiling. - fileCount: 5_693, + // 2026-07-24 (local Whisper STT): the local-origin transcription endpoint + // and its sidecar runner trace at 5,694 on Linux and 5,696 on Windows. + // Retain the established ten-file cross-platform buffer. + fileCount: 5_706, unpackedBytes: 200 * 1024 * 1024 - 1, }); diff --git a/scripts/sidecar-runtime-closure.test.mjs b/scripts/sidecar-runtime-closure.test.mjs index c26bab06d..f9189adb0 100644 --- a/scripts/sidecar-runtime-closure.test.mjs +++ b/scripts/sidecar-runtime-closure.test.mjs @@ -97,10 +97,10 @@ try { await assembleSidecarRuntime(projectRoot, standaloneRoot, dependencyRoot, destination); const metrics = await verifySidecarRuntime(destination); - assert.ok(metrics.fileCount <= 5_693); + assert.ok(metrics.fileCount <= 5_706); assert.ok(metrics.unpackedBytes < 200 * 1024 * 1024); assert.deepEqual(SIDECAR_RUNTIME_BUDGETS, { - fileCount: 5_693, + fileCount: 5_706, unpackedBytes: 200 * 1024 * 1024 - 1, }); diff --git a/scripts/sidecar-runtime-smoke.mjs b/scripts/sidecar-runtime-smoke.mjs index 370b20022..4fad71eb4 100644 --- a/scripts/sidecar-runtime-smoke.mjs +++ b/scripts/sidecar-runtime-smoke.mjs @@ -18,6 +18,13 @@ const bundledNode = path.join( "bin", process.platform === "win32" ? "node.exe" : "node", ); +const bundledWhisper = path.join( + root, + "src-tauri", + "resources", + "whisper", + process.platform === "win32" ? "whisper-cli.exe" : "whisper-cli", +); const token = "sidecar-runtime-smoke-token"; function reservePort() { @@ -104,6 +111,7 @@ function launchSidecar({ sidecarServer, sidecarRoot, covenHome, port }) { HOSTNAME: "127.0.0.1", PORT: String(port), COVEN_CAVE_BUNDLE: "1", + COVEN_WHISPER_CPP_BIN: bundledWhisper, COVEN_CAVE_AUTH_TOKEN: token, COVEN_HOME: covenHome, NEXT_TELEMETRY_DISABLED: "1", @@ -148,7 +156,7 @@ async function main() { assert.match(manifest.payloadSha256, /^[a-f0-9]{64}$/); assert.match(manifest.treeSha256, /^[a-f0-9]{64}$/); assert.match(manifest.archiveSha256, /^[a-f0-9]{64}$/); - assert.ok(manifest.fileCount > 0 && manifest.fileCount <= 5_693); + assert.ok(manifest.fileCount > 0 && manifest.fileCount <= 5_706); assert.ok(manifest.archiveBytes > 0 && manifest.archiveBytes <= 80 * 1024 * 1024); assert.ok(manifest.unpackedBytes > 0 && manifest.unpackedBytes < 200 * 1024 * 1024); extractedSidecarRoot = await mkdtemp(path.join(os.tmpdir(), "coven-cave-sidecar-archive-")); @@ -190,6 +198,24 @@ async function main() { } await access(sidecarServer); await access(bundledNode); + await access(bundledWhisper); + if (process.platform === "win32") { + for (const runtimeDll of ["MSVCP140.dll", "VCRUNTIME140.dll", "VCRUNTIME140_1.dll", "VCOMP140.dll"]) { + await access(path.join(path.dirname(bundledWhisper), runtimeDll)); + } + } + + const whisperVersion = spawnSync(bundledWhisper, ["--version"], { + encoding: "utf8", + env: process.platform === "linux" + ? { ...process.env, LD_LIBRARY_PATH: path.dirname(bundledWhisper) } + : process.env, + }); + assert.equal( + whisperVersion.status, + 0, + `packaged Whisper CLI must launch from resources: ${whisperVersion.stderr || whisperVersion.error}`, + ); const nativeModules = spawnSync( bundledNode, diff --git a/scripts/whisper-runtime-bundle.sh b/scripts/whisper-runtime-bundle.sh new file mode 100644 index 000000000..fa84c9ae0 --- /dev/null +++ b/scripts/whisper-runtime-bundle.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Stage a pinned whisper.cpp runtime beside the desktop sidecar. The output is +# a release resource, not part of the Next.js server archive: Windows keeps +# its DLLs adjacent to whisper-cli.exe and macOS signs the nested executable. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DEST="$ROOT/src-tauri/resources/whisper" +VERSION="v1.9.1" +RELEASE_URL="https://github.com/ggml-org/whisper.cpp/releases/download/$VERSION" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/coven-whisper-runtime.XXXXXX")" + +cleanup() { + rm -rf "$WORK" +} +trap cleanup EXIT + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +download_verified() { + local url="$1" + local expected="$2" + local output="$3" + curl --fail --location --proto '=https' --tlsv1.2 --retry 3 --retry-delay 2 "$url" -o "$output" + local actual + actual="$(sha256_file "$output")" + if [ "$actual" != "$expected" ]; then + echo "ERROR: Whisper runtime checksum mismatch for $url" >&2 + echo " expected $expected, got $actual" >&2 + exit 1 + fi +} + +stage_linux() { + local archive="$WORK/whisper.tar.gz" archive_name archive_sha source + case "$(uname -m)" in + x86_64|amd64) + archive_name="whisper-bin-ubuntu-x64.tar.gz" + archive_sha="f3bf3b4369a99b54665b0f19b88483b30de27f25963b0414235dea03198515c5" + ;; + aarch64|arm64) + archive_name="whisper-bin-ubuntu-arm64.tar.gz" + archive_sha="e0b66cd551ff6f2a28fabe3c6e89691eea037bb76833493abb9a71ca788994b3" + ;; + *) + echo "ERROR: no bundled Whisper runtime for Linux architecture $(uname -m)" >&2 + exit 1 + ;; + esac + download_verified \ + "$RELEASE_URL/$archive_name" \ + "$archive_sha" \ + "$archive" + tar -xzf "$archive" -C "$WORK" + source="$WORK/${archive_name%.tar.gz}" + cp -L "$source/whisper-cli" "$DEST/whisper-cli" + # Preserve both the versioned ELF files and their SONAME links (for example + # libwhisper.so.1 -> libwhisper.so.1.9.1). The binary requests the SONAME, + # not necessarily the filename selected by the archive extraction order. + find "$source" -maxdepth 1 \( -type f -o -type l \) \( -name 'libwhisper.so*' -o -name 'libggml*.so*' \) -exec cp -P {} "$DEST/" \; + test -L "$DEST/libwhisper.so.1" + chmod 755 "$DEST/whisper-cli" +} + +stage_windows() { + local archive="$WORK/whisper.zip" + download_verified \ + "$RELEASE_URL/whisper-bin-x64.zip" \ + "7d8be46ecd31828e1eb7a2ecdd0d6b314feafd82163038ab6092594b0a063539" \ + "$archive" + unzip -q "$archive" -d "$WORK/unpacked" + local source="$WORK/unpacked/Release" + cp "$source/whisper-cli.exe" "$DEST/whisper-cli.exe" + cp "$source/whisper.dll" "$DEST/whisper.dll" + find "$source" -maxdepth 1 -type f -name 'ggml*.dll' -exec cp {} "$DEST/" \; + + # whisper.cpp's official Windows build is dynamically linked to the MSVC + # CRT/OpenMP DLLs. Ship the redistributable app-locally so a fresh Windows + # install does not need a separate Visual C++ Redistributable install. + local runtime_dir="" candidate dll complete + local -a msvc_dlls=(MSVCP140.dll VCRUNTIME140.dll VCRUNTIME140_1.dll VCOMP140.dll) + for candidate in \ + "${VCToolsRedistDir:-}/x64/Microsoft.VC143.CRT" \ + "${WINDIR:-C:/Windows}/System32"; do + [ -d "$candidate" ] || continue + complete=1 + for dll in "${msvc_dlls[@]}"; do + [ -f "$candidate/$dll" ] || { complete=0; break; } + done + if [ "$complete" = "1" ]; then runtime_dir="$candidate"; break; fi + done + if [ -z "$runtime_dir" ]; then + echo "ERROR: could not find the x64 Microsoft Visual C++ runtime for bundled Whisper" >&2 + exit 1 + fi + for dll in "${msvc_dlls[@]}"; do cp "$runtime_dir/$dll" "$DEST/$dll"; done +} + +stage_macos() { + # whisper.cpp does not publish a macOS CLI archive. Build the exact immutable + # release commit on the matching release host; release.sh then signs it. + local source="$WORK/whisper.cpp" + command -v cmake >/dev/null 2>&1 || { echo "ERROR: cmake is required to build bundled Whisper on macOS" >&2; exit 1; } + git init -q "$source" + git -C "$source" remote add origin https://github.com/ggml-org/whisper.cpp.git + git -C "$source" fetch -q --depth 1 origin f049fff95a089aa9969deb009cdd4892b3e74916 + git -C "$source" checkout -q --detach FETCH_HEAD + test "$(git -C "$source" rev-parse HEAD)" = "f049fff95a089aa9969deb009cdd4892b3e74916" + # Keep copied dylibs discoverable after the temporary build tree disappears. + # The release bundle co-locates them with whisper-cli, so @loader_path is + # both relocatable and compatible with the later nested-code-signing pass. + cmake -S "$source" -B "$source/build" \ + -DWHISPER_BUILD_TESTS=OFF \ + -DWHISPER_BUILD_EXAMPLES=ON \ + -DCMAKE_BUILD_RPATH='@loader_path' \ + -DCMAKE_INSTALL_RPATH='@loader_path' \ + -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON + cmake --build "$source/build" --target whisper-cli --parallel 2 + cp "$source/build/bin/whisper-cli" "$DEST/whisper-cli" + find "$source/build/bin" -maxdepth 1 -type f -name '*.dylib' -exec cp {} "$DEST/" \; + chmod 755 "$DEST/whisper-cli" +} + +rm -rf "$DEST" +mkdir -p "$DEST" + +case "$(uname -s)" in + Linux) stage_linux ;; + Darwin) stage_macos ;; + MINGW*|MSYS*|CYGWIN*) stage_windows ;; + *) echo "ERROR: no bundled Whisper runtime for $(uname -s)" >&2; exit 1 ;; +esac + +echo "==> bundled whisper.cpp $VERSION ($(find "$DEST" -maxdepth 1 -type f | wc -l | tr -d ' ') files)" diff --git a/src-tauri/release-runtime.test.mjs b/src-tauri/release-runtime.test.mjs index c76d08923..10420b224 100644 --- a/src-tauri/release-runtime.test.mjs +++ b/src-tauri/release-runtime.test.mjs @@ -122,11 +122,12 @@ async function readNativeHost(...modules) { )).join("\n"); } -test("release bundle includes and prefers a bundled Node runtime", async () => { - const [tauriConfig, windowsConfig, bundleScript, launcher] = await Promise.all([ +test("release bundle includes and prefers bundled Node and Whisper runtimes", async () => { + const [tauriConfig, windowsConfig, bundleScript, whisperBundleScript, launcher] = await Promise.all([ readFile(new URL("./tauri.conf.json", import.meta.url), "utf8"), readFile(new URL("./tauri.windows.conf.json", import.meta.url), "utf8"), readFile(new URL("../scripts/sidecar-bundle.sh", import.meta.url), "utf8"), + readFile(new URL("../scripts/whisper-runtime-bundle.sh", import.meta.url), "utf8"), readNativeHost("sidecar_discovery.rs", "sidecar_startup.rs"), ]); @@ -135,6 +136,8 @@ test("release bundle includes and prefers a bundled Node runtime", async () => { /"resources\/node\/\*\*\/\*"/, "Tauri resources must include the bundled Node runtime", ); + assert.match(tauriConfig, /"resources\/whisper\/\*\*\/\*"/, "Tauri resources must include Whisper runtime files"); + assert.match(windowsConfig, /"resources\/whisper\/\*\*\/\*"/, "Windows bundles must include Whisper runtime files"); assert.match( windowsConfig, /"resources\/server-archive\/\*\*\/\*"/, @@ -160,6 +163,20 @@ test("release bundle includes and prefers a bundled Node runtime", async () => { /command -v node/, "sidecar bundle script must copy the release runner's Node binary", ); + assert.match(bundleScript, /whisper-runtime-bundle\.sh/, "sidecar bundle script must stage Whisper before packaging"); + assert.match(whisperBundleScript, /whisper-bin-x64\.zip/, "Windows must stage a pinned Whisper CLI archive"); + assert.match(whisperBundleScript, /MSVCP140\.dll.*VCRUNTIME140\.dll.*VCRUNTIME140_1\.dll.*VCOMP140\.dll/, "Windows must ship Whisper's app-local MSVC runtime"); + assert.match(whisperBundleScript, /whisper-bin-ubuntu-x64\.tar\.gz/, "Linux must stage a pinned Whisper CLI archive"); + assert.match( + whisperBundleScript, + /aarch64\|arm64\)[\s\S]*?whisper-bin-ubuntu-arm64\.tar\.gz[\s\S]*?e0b66cd551ff6f2a28fabe3c6e89691eea037bb76833493abb9a71ca788994b3/, + "Linux ARM64 must stage and verify its matching Whisper CLI archive", + ); + assert.match(whisperBundleScript, /f049fff95a089aa9969deb009cdd4892b3e74916/, "macOS must build the pinned Whisper release commit"); + assert.match(whisperBundleScript, /CMAKE_INSTALL_RPATH='@loader_path'/, "macOS Whisper must resolve copied dylibs relative to its executable"); + assert.doesNotMatch(whisperBundleScript, /install_name_tool -add_rpath/, "macOS Whisper must not add a duplicate CMake-provided rpath"); + assert.match(whisperBundleScript, /cp -P/, "Linux Whisper staging must preserve SONAME links"); + assert.match(whisperBundleScript, /checksum mismatch/, "Whisper artifact downloads must be hash-verified"); assert.match( launcher, /fn find_node\(resource_dir: &Path\)/, @@ -170,6 +187,9 @@ test("release bundle includes and prefers a bundled Node runtime", async () => { /resources[\s\S]*node[\s\S]*bin[\s\S]*node/, "launcher must know the bundled Node resource path", ); + assert.match(launcher, /fn find_bundled_whisper_cli\(resource_dir: &Path\)/, "launcher must resolve Whisper relative to app resources"); + assert.match(launcher, /COVEN_WHISPER_CPP_BIN/, "launcher must provide the absolute bundled Whisper path to the sidecar"); + assert.match(launcher, /LD_LIBRARY_PATH/, "Linux sidecars must load Whisper's bundled shared libraries"); assert.match( launcher, /sidecar_archive::prepare_sidecar_runtime\(app, &resource_dir\)/, @@ -184,6 +204,7 @@ test("clean release runners have resource glob placeholders", async () => { access(new URL("./resources/server/placeholder.txt", import.meta.url)), access(new URL("./resources/server-archive/placeholder.txt", import.meta.url)), access(new URL("./resources/node/placeholder.txt", import.meta.url)), + access(new URL("./resources/whisper/placeholder.txt", import.meta.url)), ]); assert.match( @@ -201,6 +222,14 @@ test("clean release runners have resource glob placeholders", async () => { /!src-tauri\/resources\/node\/placeholder\.txt/, "node placeholder must be tracked so resources/node/**/* matches in clean CI", ); + const releaseScript = await readFile(new URL("../scripts/release.sh", import.meta.url), "utf8"); + assert.match(releaseScript, /WHISPER_CLI=.*resources\/whisper\/whisper-cli/, "macOS release must resolve bundled Whisper before signing"); + assert.match(releaseScript, /"\$WHISPER_CLI" --version/, "macOS release must smoke-test the copied Whisper runtime"); + assert.match( + gitignore, + /!src-tauri\/resources\/whisper\/placeholder\.txt/, + "Whisper placeholder must be tracked so resources/whisper/**/* matches in clean CI", + ); }); test("native updater cleanup stops the sidecar before Windows exits", async () => { @@ -385,7 +414,7 @@ test("mobile startup remains available after native-host extraction", async () = assert.match(nativeHost, /\nmod tauri_setup;/); assert.doesNotMatch(nativeHost, /#\[cfg\(desktop\)\]\s*\nmod tauri_setup;/); - assert.match(setup, /#\[cfg_attr\(mobile, tauri::mobile_entry_point\)\]\npub fn run\(\)/); + assert.match(setup, /#\[cfg_attr\(mobile, tauri::mobile_entry_point\)\]\r?\npub fn run\(\)/); }); test("Windows close watchdog helper follows extracted lifecycle tests", async () => { diff --git a/src-tauri/resources/whisper/placeholder.txt b/src-tauri/resources/whisper/placeholder.txt new file mode 100644 index 000000000..7eb5b89f4 --- /dev/null +++ b/src-tauri/resources/whisper/placeholder.txt @@ -0,0 +1 @@ +generated at release build time diff --git a/src-tauri/src/sidecar_archive_manifest.rs b/src-tauri/src/sidecar_archive_manifest.rs index 39d82519b..8fbaa42c6 100644 --- a/src-tauri/src/sidecar_archive_manifest.rs +++ b/src-tauri/src/sidecar_archive_manifest.rs @@ -6,7 +6,7 @@ pub(super) const MANIFEST_SCHEMA_VERSION: u32 = 3; pub(super) const ARCHIVE_FORMAT: &str = "tar.zst"; pub(super) const MAX_ARCHIVE_BYTES: u64 = 80 * 1024 * 1024; pub(super) const MAX_UNPACKED_BYTES: u64 = 200 * 1024 * 1024 - 1; -pub(super) const MAX_FILE_COUNT: u64 = 5_693; +pub(super) const MAX_FILE_COUNT: u64 = 5_706; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] diff --git a/src-tauri/src/sidecar_discovery.rs b/src-tauri/src/sidecar_discovery.rs index 0d9503933..a619251b0 100644 --- a/src-tauri/src/sidecar_discovery.rs +++ b/src-tauri/src/sidecar_discovery.rs @@ -9,6 +9,31 @@ pub(super) fn bundled_node_path(resource_dir: &Path) -> PathBuf { .join("node.exe") } +#[cfg(all(desktop, target_os = "windows"))] +pub(super) fn bundled_whisper_cli_path(resource_dir: &Path) -> PathBuf { + resource_dir + .join("resources") + .join("whisper") + .join("whisper-cli.exe") +} + +#[cfg(all(desktop, not(target_os = "windows")))] +pub(super) fn bundled_whisper_cli_path(resource_dir: &Path) -> PathBuf { + resource_dir + .join("resources") + .join("whisper") + .join("whisper-cli") +} + +/// Release builds must use the exact whisper.cpp executable staged with the +/// app. This intentionally has no PATH fallback: a missing bundle is a +/// packaging failure, not an invitation to upload audio to a host toolchain. +#[cfg(desktop)] +pub(super) fn find_bundled_whisper_cli(resource_dir: &Path) -> Option { + let bundled = bundled_whisper_cli_path(resource_dir); + bundled.exists().then_some(bundled) +} + #[cfg(all(desktop, not(target_os = "windows")))] pub(super) fn bundled_node_path(resource_dir: &Path) -> PathBuf { resource_dir diff --git a/src-tauri/src/sidecar_startup.rs b/src-tauri/src/sidecar_startup.rs index a27c7b911..e3d17148f 100644 --- a/src-tauri/src/sidecar_startup.rs +++ b/src-tauri/src/sidecar_startup.rs @@ -140,6 +140,13 @@ pub(super) fn start_sidecar_runtime( ) })?; log::info!("[cave] using node at {}", node.display()); + let whisper_cli = find_bundled_whisper_cli(&resource_dir).ok_or_else(|| { + SidecarStartError::Failed( + "Could not find the bundled local Whisper runtime. Reinstall CovenCave or contact support." + .to_string(), + ) + })?; + log::info!("[cave] using bundled Whisper at {}", whisper_cli.display()); // Capture sidecar logs so startup failures can be surfaced in the local // preparation window instead of leaving a blank webview. @@ -240,9 +247,18 @@ pub(super) fn start_sidecar_runtime( .env("HOSTNAME", "127.0.0.1") .env("NODE_ENV", "production") .env("COVEN_CAVE_BUNDLE", "1") + .env("COVEN_WHISPER_CPP_BIN", &whisper_cli) .env("COVEN_CAVE_AUTH_TOKEN", &auth_token) .env("COVEN_CAVE_ACCESS_TOKEN", &mobile_access_token); + // Ubuntu's pinned whisper.cpp archive keeps its shared objects next to the + // CLI. Constrain the loader path to that bundled directory so the local + // runner never depends on system libraries or a developer's shell setup. + #[cfg(target_os = "linux")] + if let Some(whisper_dir) = whisper_cli.parent() { + command.env("LD_LIBRARY_PATH", whisper_dir); + } + if let Some(output) = stdout_log { command.stdout(Stdio::from(output)); } else { diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 80534da31..d393e415d 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -43,7 +43,8 @@ "longDescription": "CovenCave is the OpenCoven desktop control room for managing familiars, local agent sessions, workflows, memory, tools, and release-ready automation from one native Tauri app.", "resources": [ "resources/server/**/*", - "resources/node/**/*" + "resources/node/**/*", + "resources/whisper/**/*" ], "icon": [ "icons/32x32.png", diff --git a/src-tauri/tauri.windows.conf.json b/src-tauri/tauri.windows.conf.json index 9c1285b8b..5a93cb73b 100644 --- a/src-tauri/tauri.windows.conf.json +++ b/src-tauri/tauri.windows.conf.json @@ -3,7 +3,8 @@ "bundle": { "resources": [ "resources/server-archive/**/*", - "resources/node/**/*" + "resources/node/**/*", + "resources/whisper/**/*" ], "windows": { "wix": { diff --git a/src/app/api/api-contracts.test.ts b/src/app/api/api-contracts.test.ts index ba265db0e..eb4390b80 100644 --- a/src/app/api/api-contracts.test.ts +++ b/src/app/api/api-contracts.test.ts @@ -236,6 +236,7 @@ const contracts: RouteContract[] = [ { route: "/voice/engines/downloads", methods: ["GET", "POST"], kind: "json", readsJson: true, invalidJson: "guarded" }, { route: "/voice/engines/downloads/[jobId]", methods: ["GET"], kind: "json" }, { route: "/voice/engines/models", methods: ["DELETE"], kind: "json", readsJson: true, invalidJson: "guarded" }, + { route: "/voice/engines/whisper", methods: ["POST"], kind: "json", localOriginGuard: true }, { route: "/voice/local/chat", methods: ["POST"], kind: "json", readsJson: true }, { route: "/voice/preview", methods: ["GET"], kind: "stream" }, { route: "/voice/session", methods: ["POST"], kind: "json", readsJson: true }, diff --git a/src/app/api/voice/engines/route.ts b/src/app/api/voice/engines/route.ts index 14ac54bf0..fe9592380 100644 --- a/src/app/api/voice/engines/route.ts +++ b/src/app/api/voice/engines/route.ts @@ -1,9 +1,17 @@ import { NextResponse } from "next/server.js"; import { speechEnginesReadiness } from "../../../../lib/voice/speech-models.ts"; +import { whisperRuntimeAvailable } from "../../../../lib/voice/sidecar-whisper.ts"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; export async function GET() { - return NextResponse.json(await speechEnginesReadiness()); + const [engines, whisperAvailable] = await Promise.all([ + speechEnginesReadiness(), + whisperRuntimeAvailable(), + ]); + return NextResponse.json({ + ...engines, + runtimes: { whisper: { available: whisperAvailable } }, + }); } diff --git a/src/app/api/voice/engines/whisper/route.test.ts b/src/app/api/voice/engines/whisper/route.test.ts new file mode 100644 index 000000000..6317babc4 --- /dev/null +++ b/src/app/api/voice/engines/whisper/route.test.ts @@ -0,0 +1,48 @@ +// @ts-nocheck +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { POST } from "./route.ts"; + +function request(form: FormData, host = "localhost", bounded = true) { + return new Request(`http://${host}/api/voice/engines/whisper`, { + method: "POST", + headers: bounded ? { host, "content-length": "1024" } : { host }, + body: form, + }); +} + +test("Whisper endpoint requires a numbered PCM WAV utterance", async () => { + const missing = new FormData(); + assert.equal((await POST(request(missing))).status, 400); + + const wrongSession = new FormData(); + wrongSession.set("session", "zero"); + wrongSession.set("audio", new Blob(["wav"], { type: "audio/wav" }), "utterance.wav"); + assert.equal((await POST(request(wrongSession))).status, 400); + + const wrongType = new FormData(); + wrongType.set("session", "1"); + wrongType.set("kind", "final"); + wrongType.set("audio", new Blob(["webm"], { type: "audio/webm" }), "utterance.webm"); + const bad = await POST(request(wrongType)); + assert.equal(bad.status, 400); + assert.equal((await bad.json()).error, "invalid_audio"); +}); + +test("Whisper endpoint rejects remote and mobile-proxy origins before parsing audio", async () => { + const form = new FormData(); + form.set("session", "1"); + form.set("kind", "final"); + form.set("audio", new Blob(["wav"], { type: "audio/wav" }), "utterance.wav"); + assert.equal((await POST(request(form, "cave.example.com"))).status, 403); +}); + +test("Whisper endpoint rejects multipart bodies without a declared bound", async () => { + const form = new FormData(); + form.set("session", "1"); + form.set("kind", "final"); + form.set("audio", new Blob(["wav"], { type: "audio/wav" }), "utterance.wav"); + const response = await POST(request(form, "localhost", false)); + assert.equal(response.status, 400); + assert.equal((await response.json()).error, "invalid_audio"); +}); diff --git a/src/app/api/voice/engines/whisper/route.ts b/src/app/api/voice/engines/whisper/route.ts new file mode 100644 index 000000000..ed8854b1f --- /dev/null +++ b/src/app/api/voice/engines/whisper/route.ts @@ -0,0 +1,83 @@ +import { NextResponse } from "next/server.js"; +import { + MAX_WHISPER_WAV_BYTES, + SidecarWhisperError, + readyWhisperModel, + transcribeSidecarWav, +} from "../../../../../lib/voice/sidecar-whisper.ts"; +import { isLocalOrigin } from "../../../../../lib/server/local-origin.ts"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +function badRequest(error: string, hint: string) { + return NextResponse.json({ ok: false, error, hint }, { status: 400 }); +} + +function sessionNumber(value: FormDataEntryValue | null): number | null { + if (typeof value !== "string") return null; + const session = Number(value); + return Number.isSafeInteger(session) && session > 0 ? session : null; +} + +function eventKind(value: FormDataEntryValue | null): "partial" | "final" | null { + return value === "partial" || value === "final" ? value : null; +} + +/** Transcribe one browser-captured PCM WAV with the first verified local model. */ +export async function POST(req: Request) { + if (!isLocalOrigin(req)) { + return NextResponse.json({ ok: false, error: "local_origin_required" }, { status: 403 }); + } + const contentLengthHeader = req.headers.get("content-length"); + // formData() buffers and parses the entire multipart body. Browsers sending + // a FormData WAV provide this header; reject unknown/chunked bodies rather + // than allowing a local sidecar request to bypass its memory ceiling. + if (!contentLengthHeader || !/^(?:0|[1-9]\d*)$/.test(contentLengthHeader)) { + return badRequest("invalid_audio", "Local Whisper requires a bounded audio upload."); + } + const contentLength = Number(contentLengthHeader); + if (!Number.isSafeInteger(contentLength) || contentLength > MAX_WHISPER_WAV_BYTES + 128 * 1024) { + return badRequest("invalid_audio", "The recorded utterance is too large for local Whisper."); + } + let form: FormData; + try { form = await req.formData(); } catch { + return badRequest("invalid_form", "Send one WAV utterance as multipart form data."); + } + const session = sessionNumber(form.get("session")); + if (session === null) return badRequest("invalid_session", "A positive speech session number is required."); + const kind = eventKind(form.get("kind")); + if (!kind) return badRequest("invalid_kind", "A speech event must be partial or final."); + const audio = form.get("audio"); + if (!(audio instanceof Blob)) return badRequest("missing_audio", "Record an audio utterance before sending it to local Whisper."); + if (audio.type !== "audio/wav") return badRequest("invalid_audio", "Local Whisper accepts PCM WAV utterances only."); + if (audio.size === 0 || audio.size > MAX_WHISPER_WAV_BYTES) { + return badRequest("invalid_audio", "The recorded utterance is too large for local Whisper."); + } + const model = await readyWhisperModel(); + if (!model) { + return NextResponse.json({ + ok: false, + error: "whisper_model_not_ready", + hint: "Download a Whisper model in Settings before using local voice.", + }, { status: 409 }); + } + const lang = form.get("lang"); + try { + const text = await transcribeSidecarWav( + new Uint8Array(await audio.arrayBuffer()), + model, + { + lang: typeof lang === "string" && /^[A-Za-z]{2,3}(?:-[A-Za-z]{2})?$/.test(lang) ? lang : undefined, + signal: req.signal, + }, + ); + return NextResponse.json({ ok: true, session, kind, text }); + } catch (error) { + const whisperError = error instanceof SidecarWhisperError + ? error + : new SidecarWhisperError("whisper_failed", "Local Whisper could not transcribe that utterance. Try again."); + const status = whisperError.code === "whisper_empty" ? 422 : whisperError.code === "whisper_unavailable" ? 503 : 502; + return NextResponse.json({ ok: false, error: whisperError.code, hint: whisperError.hint }, { status }); + } +} diff --git a/src/components/voice-call-overlay.tsx b/src/components/voice-call-overlay.tsx index c4547aa32..34f5967c0 100644 --- a/src/components/voice-call-overlay.tsx +++ b/src/components/voice-call-overlay.tsx @@ -300,6 +300,7 @@ export function VoiceCallOverlay({ familiar, sessionId, onClose }: Props) { // promise kept; the other modes tell the user their audio rides a service. function earsEngineLabel(engine: VoiceEarsEngine): string { switch (engine) { + case "sidecar-whisper": return "Hearing via local Whisper"; case "native-on-device": return "Hearing on-device"; case "native-dictation": return "Hearing via Apple dictation"; case "web-speech": return "Hearing via browser speech"; diff --git a/src/lib/voice/dictation-controller.ts b/src/lib/voice/dictation-controller.ts index 0d72507e1..b0a194ba0 100644 --- a/src/lib/voice/dictation-controller.ts +++ b/src/lib/voice/dictation-controller.ts @@ -30,7 +30,9 @@ export type DictationController = { * Apple's dictation service (requireOnDevice false): it is the OS-level * dictation UX users already expect. */ export async function resolveDictationEars(): Promise { - const preferred = await resolvePreferredEars().catch(() => undefined); + // Composer dictation does not own a MediaStream, so use its existing native + // or browser engines rather than the call-only Whisper capture adapter. + const preferred = await resolvePreferredEars({ allowSidecar: false }).catch(() => undefined); if (preferred) return preferred.factory; return createWebSpeechEars(); } diff --git a/src/lib/voice/native-stt.test.ts b/src/lib/voice/native-stt.test.ts index bcff00f1f..10334d176 100644 --- a/src/lib/voice/native-stt.test.ts +++ b/src/lib/voice/native-stt.test.ts @@ -5,6 +5,7 @@ import { createNativeSttEars, nativeSttAvailable, nativeSttAvailability, + resolvePreferredEars, selectNativeEarsEngine, STT_EVENT, } from "./native-stt.ts"; @@ -334,3 +335,61 @@ test("engine selection: on-device wins, strict refuses dictation, fallback is la /Dictation/.test(err.hint), ); }); + +test("a ready sidecar Whisper model outranks the native bridge", async () => { + const priorWindow = globalThis.window; + const priorFetch = globalThis.fetch; + globalThis.window = { location: { hostname: "localhost" } }; + globalThis.fetch = async () => new Response(JSON.stringify({ + ok: true, + runtimes: { whisper: { available: true } }, + stt: [{ engine: "whisper", ready: true }], + })); + try { + const preferred = await resolvePreferredEars({ requireOnDevice: true }); + assert.equal(preferred?.engine, "sidecar-whisper"); + } finally { + globalThis.window = priorWindow; + globalThis.fetch = priorFetch; + } +}); + +test("English-only sidecar Whisper does not satisfy Local voice for a non-English locale", async () => { + const priorWindow = globalThis.window; + const priorFetch = globalThis.fetch; + const priorNavigator = Object.getOwnPropertyDescriptor(globalThis, "navigator"); + globalThis.window = { location: { hostname: "localhost" } }; + Object.defineProperty(globalThis, "navigator", { configurable: true, value: { language: "fr-FR" } }); + globalThis.fetch = async () => new Response(JSON.stringify({ + ok: true, + runtimes: { whisper: { available: true } }, + stt: [{ id: "whisper-tiny-en", engine: "whisper", ready: true }], + })); + try { + await assert.rejects( + () => resolvePreferredEars({ requireOnDevice: true }), + (error) => error?.name === "VoiceConnectError" && error.message === "stt_on_device_unavailable", + ); + } finally { + globalThis.window = priorWindow; + globalThis.fetch = priorFetch; + if (priorNavigator) Object.defineProperty(globalThis, "navigator", priorNavigator); + else delete globalThis.navigator; + } +}); + +test("strict Local voice rejects when neither sidecar nor native STT is available", async () => { + const priorWindow = globalThis.window; + const priorFetch = globalThis.fetch; + globalThis.window = { __TAURI_INTERNALS__: {}, location: { hostname: "localhost" } }; + globalThis.fetch = async () => new Response("unavailable", { status: 503 }); + try { + await assert.rejects( + () => resolvePreferredEars({ requireOnDevice: true }), + (error) => error?.name === "VoiceConnectError" && error.message === "stt_on_device_unavailable", + ); + } finally { + globalThis.window = priorWindow; + globalThis.fetch = priorFetch; + } +}); diff --git a/src/lib/voice/native-stt.ts b/src/lib/voice/native-stt.ts index 009743a84..06ec4c508 100644 --- a/src/lib/voice/native-stt.ts +++ b/src/lib/voice/native-stt.ts @@ -17,6 +17,7 @@ import type { SpeechEars, SpeechEarsFactory, SpeechEarsHandlers } from "./speech-loop.ts"; import type { VoiceEarsEngine } from "./types.ts"; import { VoiceConnectError } from "./types.ts"; +import { createSidecarWhisperEars, localSidecarWhisperAvailable } from "./sidecar-whisper-ears.ts"; /** Event channel mirrored from src-tauri/src/speech.rs. */ export const STT_EVENT = "speech-stt:event"; @@ -234,7 +235,7 @@ export function createNativeSttEars( /** Preferred ears plus which engine mode they run on (for honest call UI). */ export type PreferredEars = { factory: SpeechEarsFactory; - engine: Extract; + engine: Extract; }; /** The hybrid on-device policy (cave-vpe1) as a pure, pinnable decision: @@ -262,13 +263,36 @@ export function selectNativeEarsEngine( * dictation service. */ export async function resolvePreferredEars( - opts: { requireOnDevice?: boolean } = {}, + opts: { requireOnDevice?: boolean; allowSidecar?: boolean } = {}, ): Promise { - const bridge = await loadNativeSttBridge(); - if (!bridge) return undefined; const lang = typeof navigator !== "undefined" ? navigator.language || undefined : undefined; + // A verified sidecar model is fully local on every supported platform, and + // deliberately outranks the Apple-specific recognizer only when it supports + // the current locale. Do not probe this during SSR: the browser's + // authenticated fetch is what reaches a packaged sidecar. + if (opts.allowSidecar !== false && typeof window !== "undefined" && await localSidecarWhisperAvailable(fetch, lang)) { + return { factory: createSidecarWhisperEars(), engine: "sidecar-whisper" }; + } + const bridge = await loadNativeSttBridge(); + if (!bridge) { + if (opts.requireOnDevice) { + throw new VoiceConnectError( + "stt_on_device_unavailable", + "Download a local Whisper model in Settings to use Local voice on this device.", + ); + } + return undefined; + } const availability = await nativeSttAvailability(bridge, lang); - if (availability?.supported !== true) return undefined; + if (availability?.supported !== true) { + if (opts.requireOnDevice) { + throw new VoiceConnectError( + "stt_on_device_unavailable", + "Download a local Whisper model in Settings to use Local voice on this device.", + ); + } + return undefined; + } const engine = selectNativeEarsEngine(availability, opts.requireOnDevice ?? false); return { factory: createNativeSttEars(bridge, { diff --git a/src/lib/voice/sidecar-whisper-ears.ts b/src/lib/voice/sidecar-whisper-ears.ts new file mode 100644 index 000000000..e37f29d15 --- /dev/null +++ b/src/lib/voice/sidecar-whisper-ears.ts @@ -0,0 +1,438 @@ +// Browser-side ears for the local whisper.cpp sidecar. +// +// whisper.cpp's CLI has a stable WAV input contract, so capture is PCM via +// WebAudio rather than MediaRecorder's browser-specific WebM/MP4 codecs. The +// endpoint is deliberately one utterance at a time: local VAD endpointing +// keeps the sidecar CPU work bounded and gives the speech loop the same +// listen -> final -> restart cadence as native-stt. + +import type { SpeechEars, SpeechEarsFactory, SpeechEarsHandlers } from "./speech-loop.ts"; + +export const SIDECAR_WHISPER_ENDPOINT = "/api/voice/engines/whisper"; +export const WHISPER_SILENCE_MS = 1_200; +export const WHISPER_MAX_UTTERANCE_MS = 30_000; +export const WHISPER_VOICE_THRESHOLD = 0.012; +export const WHISPER_SAMPLE_RATE = 16_000; +export const WHISPER_PARTIAL_MS = 600; + +type EnginesPayload = { + ok?: boolean; + stt?: Array<{ id?: string; engine?: string; ready?: boolean }>; + runtimes?: { whisper?: { available?: boolean } }; +}; + +type WhisperResponse = { + ok?: boolean; + session?: number; + kind?: "partial" | "final"; + text?: string; + error?: string; + hint?: string; +}; + +type TimerOptions = { + stabilityMs?: number; + maxUtteranceMs?: number; + voiceThreshold?: number; + fetchImpl?: typeof fetch; + setTimeout?: (fn: () => void, ms: number) => unknown; + clearTimeout?: (handle: unknown) => void; +}; + +type AudioContextConstructor = new () => AudioContext; + +/** English-only Whisper models must not displace a recognizer that supports + * the user's non-English locale. Multilingual models may auto-detect it. */ +export function whisperModelSupportsLocale(modelId: string | undefined, locale?: string): boolean { + const language = locale?.trim().split("-")[0]?.toLowerCase(); + if (!language || language === "en") return true; + if (!modelId) return false; + return !/(?:[-_.]en)$/i.test(modelId); +} + +/** True only for a verified downloaded Whisper model advertised by the sidecar + * that can transcribe the requested locale. */ +export async function sidecarWhisperAvailable(fetchImpl: typeof fetch = fetch, locale?: string): Promise { + try { + const res = await fetchImpl("/api/voice/engines", { cache: "no-store" }); + if (!res.ok) return false; + const payload = await res.json() as EnginesPayload; + return payload.ok === true && payload.runtimes?.whisper?.available === true && payload.stt?.some( + (model) => model.engine === "whisper" && model.ready === true && whisperModelSupportsLocale(model.id, locale), + ) === true; + } catch { + return false; + } +} + +/** Mobile Tauri deliberately uses a remote daemon instead of a local Node + * sidecar. Never treat that remote audio hop as the Local provider's strict + * on-device engine. */ +async function hasLocalDesktopSidecar(): Promise { + if (typeof window === "undefined") return false; + if (!(window as unknown as Record).__TAURI_INTERNALS__) { + // A hosted Cave can proxy the same API shape, but posting microphone audio + // there is not local STT. Browser support is intentionally loopback-only. + const host = window.location.hostname; + return host === "localhost" || host === "127.0.0.1" || host === "[::1]"; + } + try { + const { platform } = await import("@tauri-apps/plugin-os"); + const value = platform(); + return value !== "ios" && value !== "android"; + } catch { + // Mobile builds register the OS plugin. Retain compatibility for older + // desktop builds where it did not yet exist. + return true; + } +} + +export async function localSidecarWhisperAvailable(fetchImpl: typeof fetch = fetch, locale?: string): Promise { + return (await hasLocalDesktopSidecar()) && sidecarWhisperAvailable(fetchImpl, locale); +} + +/** Encode mono float PCM into the 16-bit WAV accepted directly by whisper.cpp. */ +export function encodePcmWav(chunks: readonly Float32Array[], sampleRate: number): Uint8Array { + const frames = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const bytes = new Uint8Array(44 + frames * 2); + const view = new DataView(bytes.buffer); + const writeAscii = (offset: number, value: string) => { + for (let index = 0; index < value.length; index++) view.setUint8(offset + index, value.charCodeAt(index)); + }; + writeAscii(0, "RIFF"); + view.setUint32(4, 36 + frames * 2, true); + writeAscii(8, "WAVE"); + writeAscii(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + writeAscii(36, "data"); + view.setUint32(40, frames * 2, true); + let offset = 44; + for (const chunk of chunks) { + for (const value of chunk) { + const clamped = Math.max(-1, Math.min(1, value)); + view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true); + offset += 2; + } + } + return bytes; +} + +/** whisper.cpp accepts 16 kHz WAV input. Browser contexts normally run at + * 44.1/48 kHz, so downsample the captured mono PCM before serialization. */ +export function resampleMonoPcm( + chunks: readonly Float32Array[], + sourceRate: number, + targetRate: number = WHISPER_SAMPLE_RATE, +): Float32Array { + const inputLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const input = new Float32Array(inputLength); + let offset = 0; + for (const chunk of chunks) { input.set(chunk, offset); offset += chunk.length; } + if (sourceRate === targetRate || inputLength === 0) return input; + const output = new Float32Array(Math.max(1, Math.round(inputLength * targetRate / sourceRate))); + for (let index = 0; index < output.length; index++) { + const sourceIndex = index * sourceRate / targetRate; + const lower = Math.floor(sourceIndex); + const upper = Math.min(lower + 1, input.length - 1); + const fraction = sourceIndex - lower; + output[index] = input[lower] * (1 - fraction) + input[upper] * fraction; + } + return output; +} + +function rms(samples: Float32Array): number { + if (samples.length === 0) return 0; + let sum = 0; + for (const sample of samples) sum += sample * sample; + return Math.sqrt(sum / samples.length); +} + +function audioContextConstructor(): AudioContextConstructor | null { + if (typeof window === "undefined") return null; + const legacy = window as unknown as { webkitAudioContext?: AudioContextConstructor }; + return window.AudioContext ?? legacy.webkitAudioContext ?? null; +} + +/** + * Ears factory selected before the loop connects. The microphone is supplied + * by connectSpeechLoop so this adapter neither requests a second permission + * nor owns track lifetime. + */ +export function createSidecarWhisperEars(options: TimerOptions = {}): SpeechEarsFactory { + const stabilityMs = options.stabilityMs ?? WHISPER_SILENCE_MS; + const maxUtteranceMs = options.maxUtteranceMs ?? WHISPER_MAX_UTTERANCE_MS; + const threshold = options.voiceThreshold ?? WHISPER_VOICE_THRESHOLD; + const request = options.fetchImpl ?? fetch; + const schedule = options.setTimeout ?? ((fn: () => void, ms: number) => setTimeout(fn, ms)); + const unschedule = options.clearTimeout ?? ((handle: unknown) => clearTimeout(handle as number)); + + return (handlers: SpeechEarsHandlers, mic?: MediaStream): SpeechEars => { + let wanted = false; + let closed = false; + let current = 0; + let counter = 0; + let stabilityTimer: unknown = null; + let initialSilenceTimer: unknown = null; + let utteranceCapTimer: unknown = null; + let partialTimer: unknown = null; + let controller: AbortController | null = null; + let partialController: AbortController | null = null; + let partialInFlight = false; + let context: AudioContext | null = null; + let source: MediaStreamAudioSourceNode | null = null; + let lowPass: BiquadFilterNode | null = null; + let processor: ScriptProcessorNode | null = null; + let silentOutput: GainNode | null = null; + let chunks: Float32Array[] = []; + let hasSpeech = false; + let finalizing = false; + let contextRelease: Promise | null = null; + + const clearTimers = () => { + if (stabilityTimer !== null) { unschedule(stabilityTimer); stabilityTimer = null; } + if (initialSilenceTimer !== null) { unschedule(initialSilenceTimer); initialSilenceTimer = null; } + if (utteranceCapTimer !== null) { unschedule(utteranceCapTimer); utteranceCapTimer = null; } + if (partialTimer !== null) { unschedule(partialTimer); partialTimer = null; } + }; + + const releaseCapture = () => { + processor?.disconnect(); + source?.disconnect(); + lowPass?.disconnect(); + silentOutput?.disconnect(); + processor = null; + source = null; + lowPass = null; + silentOutput = null; + const oldContext = context; + context = null; + if (!oldContext || oldContext.state === "closed") return; + const release = oldContext.close().catch(() => { /* teardown */ }); + contextRelease = release; + void release.finally(() => { + if (contextRelease === release) contextRelease = null; + }); + }; + + const restart = () => { + if (wanted && !closed && !finalizing) start(); + }; + + const armPartial = (session: number) => { + if ( + closed || !wanted || finalizing || current !== session || chunks.length === 0 || + partialInFlight || partialTimer !== null + ) return; + partialTimer = schedule(() => { + partialTimer = null; + sendPartial(session); + }, WHISPER_PARTIAL_MS); + }; + + const postAudio = ( + session: number, + audio: readonly Float32Array[], + sampleRate: number, + kind: "partial" | "final", + signal: AbortSignal, + ) => { + const wav = encodePcmWav([resampleMonoPcm(audio, sampleRate)], WHISPER_SAMPLE_RATE); + const form = new FormData(); + form.set("session", String(session)); + form.set("kind", kind); + const wavBuffer = wav.buffer.slice(wav.byteOffset, wav.byteOffset + wav.byteLength) as ArrayBuffer; + form.set("audio", new Blob([wavBuffer], { type: "audio/wav" }), "utterance.wav"); + return request(SIDECAR_WHISPER_ENDPOINT, { method: "POST", body: form, signal }) + .then(async (res) => ({ res, body: await res.json().catch(() => null) as WhisperResponse | null })); + }; + + const sendPartial = (session: number) => { + if (closed || !wanted || finalizing || current !== session || partialInFlight || chunks.length === 0) return; + const partialAudio = chunks.slice(); + const sampleRate = context?.sampleRate ?? WHISPER_SAMPLE_RATE; + const requestController = new AbortController(); + partialController = requestController; + partialInFlight = true; + void postAudio(session, partialAudio, sampleRate, "partial", requestController.signal) + .then(({ res, body }) => { + if (closed || requestController.signal.aborted || finalizing || current !== session) return; + if (!res.ok || !body?.ok || body.session !== session || body.kind !== "partial") { + // An early incremental decode can legitimately contain no complete + // words yet. Keep collecting; the final decode remains authoritative. + if (body?.error === "whisper_empty") return; + wanted = false; + stop(); + handlers.onError(body?.error ?? "whisper_failed", body?.hint); + return; + } + const text = body.text?.trim(); + if (text) handlers.onPartial(text); + }) + .catch((error) => { + if (closed || requestController.signal.aborted) return; + wanted = false; + stop(); + handlers.onError("whisper_unavailable", error instanceof Error ? error.message : String(error)); + }) + .finally(() => { + partialInFlight = false; + if (partialController === requestController) partialController = null; + armPartial(session); + }); + }; + + const finish = (session: number) => { + if (closed || !wanted || finalizing || session !== current) return; + clearTimers(); + finalizing = true; + partialController?.abort(); + partialController = null; + const sampleRate = context?.sampleRate ?? 16_000; + releaseCapture(); + const audio = chunks; + chunks = []; + if (!hasSpeech || audio.length === 0) { + current = 0; + finalizing = false; + restart(); + return; + } + hasSpeech = false; + const requestController = new AbortController(); + controller = requestController; + void postAudio(session, audio, sampleRate, "final", requestController.signal) + .then(({ res, body }) => { + if (closed || requestController.signal.aborted || !wanted || !finalizing || current !== session) return; + if (!res.ok || !body?.ok || body.session !== session || body.kind !== "final") { + // Match native STT's empty-final behavior: resume listening rather + // than turning a pause or an unrecognized sound into a call error. + if (body?.error === "whisper_empty") { + current = 0; + finalizing = false; + restart(); + return; + } + wanted = false; + stop(); + handlers.onError(body?.error ?? "whisper_failed", body?.hint); + return; + } + const text = body.text?.trim(); + current = 0; + finalizing = false; + if (text) handlers.onFinal(text); + restart(); + }) + .catch((error) => { + if (closed || requestController.signal.aborted) return; + wanted = false; + stop(); + handlers.onError("whisper_unavailable", error instanceof Error ? error.message : String(error)); + }) + .finally(() => { + if (controller === requestController) controller = null; + }); + }; + + const start = () => { + if (closed || !wanted || finalizing || current !== 0) return; + // AudioContext.close() releases hardware asynchronously. Waiting here + // prevents rapid hush/listen and empty-utterance restarts from briefly + // exceeding browsers' active-context limit. + if (contextRelease) { + void contextRelease.then(start, start); + return; + } + if (!mic) { + wanted = false; + handlers.onError("whisper_unavailable", "The local Whisper engine could not access the microphone stream."); + return; + } + const Context = audioContextConstructor(); + if (!Context) { + wanted = false; + handlers.onError("whisper_unavailable", "This browser cannot capture PCM audio for local Whisper."); + return; + } + const session = ++counter; + current = session; + chunks = []; + hasSpeech = false; + finalizing = false; + context = new Context(); + source = context.createMediaStreamSource(mic); + // Filter before decimation so high-frequency microphone noise cannot + // alias into Whisper's 0–8 kHz speech band during PCM downsampling. + lowPass = context.createBiquadFilter(); + lowPass.type = "lowpass"; + lowPass.frequency.value = Math.min(7_200, context.sampleRate / 2 - 100); + lowPass.Q.value = 0.707; + processor = context.createScriptProcessor(4_096, 1, 1); + // A zero-gain output keeps ScriptProcessor active without feeding the mic + // back to the speakers. + silentOutput = context.createGain(); + silentOutput.gain.value = 0; + processor.onaudioprocess = (event) => { + if (closed || current !== session) return; + const input = event.inputBuffer.getChannelData(0); + if (rms(input) < threshold) { + // Initial silence is discarded by finish() and never needs a PCM + // copy. Once speech starts, retain trailing silence for endpointing. + if (hasSpeech) chunks.push(input.slice()); + return; + } + const chunk = input.slice(); + if (!hasSpeech) { + hasSpeech = true; + // The initial-silence timer may have retained PCM while waiting for + // VAD. Drop it when speech begins so the utterance cap bounds the + // actual WAV sent to Whisper, rather than silence plus speech. + chunks = [chunk]; + if (initialSilenceTimer !== null) { unschedule(initialSilenceTimer); initialSilenceTimer = null; } + // The full cap applies to the utterance, not to time spent waiting + // for someone to start speaking. + utteranceCapTimer = schedule(() => finish(session), maxUtteranceMs); + } else { + chunks.push(chunk); + } + if (stabilityTimer !== null) unschedule(stabilityTimer); + stabilityTimer = schedule(() => finish(session), stabilityMs); + armPartial(session); + }; + source.connect(lowPass); + lowPass.connect(processor); + processor.connect(silentOutput); + silentOutput.connect(context.destination); + // Bound initial silence and a failed/quiet microphone. The empty path in + // finish() discards its PCM and reopens listening without a network call. + // Once VAD detects speech, this is replaced with a full utterance cap. + initialSilenceTimer = schedule(() => finish(session), maxUtteranceMs); + void context.resume().catch(() => { /* user gesture is supplied by call start */ }); + }; + + const stop = () => { + clearTimers(); + current = 0; + finalizing = false; + releaseCapture(); + chunks = []; + hasSpeech = false; + controller?.abort(); + controller = null; + partialController?.abort(); + partialController = null; + }; + + return { + listen() { wanted = true; start(); }, + hush() { wanted = false; stop(); }, + close() { closed = true; wanted = false; stop(); }, + }; + }; +} diff --git a/src/lib/voice/sidecar-whisper.test.ts b/src/lib/voice/sidecar-whisper.test.ts new file mode 100644 index 000000000..4656486ed --- /dev/null +++ b/src/lib/voice/sidecar-whisper.test.ts @@ -0,0 +1,355 @@ +// @ts-nocheck +import assert from "node:assert/strict"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { test } from "node:test"; +import { + MAX_WHISPER_WAV_BYTES, + SidecarWhisperError, + createReadyWhisperModelCache, + transcribeSidecarWav, + whisperCliArgs, + whisperCliCommand, +} from "./sidecar-whisper.ts"; +import { + createSidecarWhisperEars, + encodePcmWav, + resampleMonoPcm, + sidecarWhisperAvailable, + WHISPER_SAMPLE_RATE, + whisperModelSupportsLocale, +} from "./sidecar-whisper-ears.ts"; + +const cacheRoot = path.join(process.cwd(), "node_modules", ".cache", "coven-cave-tests", "sidecar-whisper"); + +test("the sidecar selects only an explicit Whisper runtime override", () => { + assert.equal(whisperCliCommand({}), "whisper-cli"); + assert.equal(whisperCliCommand({ COVEN_WHISPER_CPP_BIN: " /opt/whisper-cli " }), "/opt/whisper-cli"); + assert.deepEqual( + whisperCliArgs("model.bin", "input.wav", "output", "en-US"), + ["-m", "model.bin", "-f", "input.wav", "-otxt", "-of", "output", "-l", "en-US"], + ); +}); + +test("verified Whisper model readiness is cached until file metadata changes", async () => { + let loads = 0; + let metadata = { size: 77, mtimeMs: 1, isFile: () => true }; + const ready = createReadyWhisperModelCache( + async () => { + loads += 1; + return { + stt: [{ id: "whisper-tiny-en", name: "Whisper tiny.en", path: "/models/tiny.bin", engine: "whisper", ready: true }], + }; + }, + async () => metadata, + ); + assert.equal((await ready())?.path, "/models/tiny.bin"); + assert.equal((await ready())?.path, "/models/tiny.bin"); + assert.equal(loads, 1, "partials reuse the verified model without rehashing it"); + metadata = { ...metadata, mtimeMs: 2 }; + await ready(); + assert.equal(loads, 2, "changed model metadata triggers a fresh verification"); +}); + +test("transcription stages a WAV privately, reads whisper.cpp text, and removes it", async () => { + await mkdir(cacheRoot, { recursive: true }); + let seenArgs = []; + const text = await transcribeSidecarWav(new Uint8Array([1, 2, 3]), { + id: "whisper-tiny-en", name: "Whisper tiny.en", path: "model.bin", + }, { + tempRoot: cacheRoot, + run: async (_command, args) => { + seenArgs = args; + const wav = await readFile(args[3]); + assert.deepEqual([...wav], [1, 2, 3]); + await writeFile(`${args[args.indexOf("-of") + 1]}.txt`, " hello from Whisper \n"); + }, + }); + assert.equal(text, "hello from Whisper"); + assert.deepEqual(seenArgs.slice(0, 6), ["-m", "model.bin", "-f", seenArgs[3], "-otxt", "-of"]); + assert.deepEqual(await (await import("node:fs/promises")).readdir(cacheRoot), []); + await rm(cacheRoot, { recursive: true, force: true }); +}); + +test("transcription rejects oversized input before creating a subprocess", async () => { + await assert.rejects( + () => transcribeSidecarWav(new Uint8Array(MAX_WHISPER_WAV_BYTES + 1), { + id: "whisper-tiny-en", name: "Whisper tiny.en", path: "model.bin", + }), + (error) => error instanceof SidecarWhisperError && error.code === "whisper_failed", + ); +}); + +test("transcription forwards cancellation to its Whisper runner", async () => { + const controller = new AbortController(); + let receivedSignal; + await mkdir(cacheRoot, { recursive: true }); + await assert.rejects( + () => transcribeSidecarWav(new Uint8Array([1]), { + id: "whisper-tiny-en", name: "Whisper tiny.en", path: "model.bin", + }, { + tempRoot: cacheRoot, + signal: controller.signal, + run: async (_command, _args, signal) => { + receivedSignal = signal; + throw new SidecarWhisperError("whisper_failed", "cancelled"); + }, + }), + /whisper_failed/, + ); + assert.equal(receivedSignal, controller.signal); + await rm(cacheRoot, { recursive: true, force: true }); +}); + +test("PCM capture serializes as a standard mono 16-bit WAV", () => { + const wav = encodePcmWav([new Float32Array([-1, 0, 1])], 16_000); + const view = new DataView(wav.buffer); + assert.equal(new TextDecoder().decode(wav.slice(0, 4)), "RIFF"); + assert.equal(new TextDecoder().decode(wav.slice(8, 12)), "WAVE"); + assert.equal(view.getUint32(24, true), 16_000); + assert.equal(view.getUint16(22, true), 1); + assert.equal(view.getInt16(44, true), -32_768); + assert.equal(view.getInt16(48, true), 32_767); +}); + +test("browser PCM is downsampled to whisper.cpp's 16 kHz input", () => { + const output = resampleMonoPcm([new Float32Array([0, 0.5, 1, 0.5])], 32_000); + assert.equal(output.length, 2); + assert.deepEqual([...output], [0, 1]); +}); + +test("sidecar availability requires a verified ready Whisper model", async () => { + assert.equal(await sidecarWhisperAvailable(async () => new Response(JSON.stringify({ + ok: true, runtimes: { whisper: { available: true } }, stt: [{ id: "whisper-tiny-en", engine: "whisper", ready: true }], + }))), true); + assert.equal(await sidecarWhisperAvailable(async () => new Response(JSON.stringify({ + ok: true, runtimes: { whisper: { available: true } }, stt: [{ engine: "whisper", ready: false }], + }))), false); + assert.equal(await sidecarWhisperAvailable(async () => new Response(JSON.stringify({ + ok: true, runtimes: { whisper: { available: false } }, stt: [{ engine: "whisper", ready: true }], + }))), false); + assert.equal(await sidecarWhisperAvailable(async () => new Response("nope", { status: 503 })), false); +}); + +test("English-only Whisper models do not claim non-English locales", async () => { + const englishModel = async () => new Response(JSON.stringify({ + ok: true, runtimes: { whisper: { available: true } }, stt: [{ id: "whisper-base-en", engine: "whisper", ready: true }], + })); + assert.equal(await sidecarWhisperAvailable(englishModel, "en-US"), true); + assert.equal(await sidecarWhisperAvailable(englishModel, "fr-FR"), false); + assert.equal(whisperModelSupportsLocale("whisper-small", "fr-FR"), true, "multilingual models may auto-detect French"); +}); + +function fakeTimers() { + let next = 0; + const pending = new Map(); + return { + setTimeout(fn, ms) { const id = ++next; pending.set(id, { fn, ms }); return id; }, + clearTimeout(id) { pending.delete(id); }, + pendingIds(ms) { + return [...pending.entries()].filter(([, value]) => value.ms === ms).map(([id]) => id); + }, + fire(ms) { + const entry = [...pending.entries()].find(([, value]) => value.ms === ms); + assert.ok(entry, `missing timer ${ms}`); + pending.delete(entry[0]); + entry[1].fn(); + }, + }; +} + +test("initial silence is capped, discarded, and restarted without a Whisper request", async () => { + const priorWindow = globalThis.window; + const timers = fakeTimers(); + const processors = []; + const filters = []; + let closes = 0; + class AudioContext { + sampleRate = 48_000; + state = "running"; + destination = {}; + createMediaStreamSource() { return { connect() {}, disconnect() {} }; } + createBiquadFilter() { + const filter = { + connect() {}, disconnect() {}, type: "", frequency: { value: 0 }, Q: { value: 0 }, + }; + filters.push(filter); + return filter; + } + createScriptProcessor() { + const processor = { connect() {}, disconnect() {}, onaudioprocess: null }; + processors.push(processor); + return processor; + } + createGain() { return { connect() {}, disconnect() {}, gain: { value: 1 } }; } + resume() { return Promise.resolve(); } + close() { closes += 1; this.state = "closed"; return Promise.resolve(); } + } + globalThis.window = { AudioContext }; + let requests = 0; + const ears = createSidecarWhisperEars({ + maxUtteranceMs: 99, + fetchImpl: async () => { requests += 1; return new Response("{}"); }, + setTimeout: timers.setTimeout, + clearTimeout: timers.clearTimeout, + })({ onPartial() {}, onFinal() {}, onError() {} }, {}); + try { + ears.listen(); + assert.equal(filters[0].type, "lowpass"); + assert.equal(filters[0].frequency.value, 7_200, "filter is applied before 16 kHz downsampling"); + processors[0].onaudioprocess({ inputBuffer: { getChannelData: () => new Float32Array(4096) } }); + timers.fire(99); + assert.equal(closes, 1, "silent capture releases its first audio context"); + assert.equal(requests, 0, "silent capture never submits a WAV"); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(processors.length, 2, "ears restart listening after the empty cap"); + } finally { + ears.close(); + globalThis.window = priorWindow; + } +}); + +test("rapid hush/listen waits for AudioContext close before opening capture again", async () => { + const priorWindow = globalThis.window; + const processors = []; + let finishClose; + class AudioContext { + sampleRate = 48_000; + state = "running"; + destination = {}; + createMediaStreamSource() { return { connect() {}, disconnect() {} }; } + createBiquadFilter() { return { connect() {}, disconnect() {}, type: "", frequency: { value: 0 }, Q: { value: 0 } }; } + createScriptProcessor() { const processor = { connect() {}, disconnect() {}, onaudioprocess: null }; processors.push(processor); return processor; } + createGain() { return { connect() {}, disconnect() {}, gain: { value: 1 } }; } + resume() { return Promise.resolve(); } + close() { + this.state = "closed"; + return new Promise((resolve) => { finishClose = resolve; }); + } + } + globalThis.window = { AudioContext }; + const ears = createSidecarWhisperEars()({ onPartial() {}, onFinal() {}, onError() {} }, {}); + try { + ears.listen(); + assert.equal(processors.length, 1); + ears.hush(); + ears.listen(); + assert.equal(processors.length, 1, "replacement capture waits for the prior context to close"); + finishClose(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(processors.length, 2, "capture restarts once the prior context is released"); + } finally { + ears.close(); + globalThis.window = priorWindow; + } +}); + +test("voice activity receives a full utterance cap after initial silence", () => { + const priorWindow = globalThis.window; + const timers = fakeTimers(); + const processors = []; + class AudioContext { + sampleRate = 48_000; + state = "running"; + destination = {}; + createMediaStreamSource() { return { connect() {}, disconnect() {} }; } + createBiquadFilter() { return { connect() {}, disconnect() {}, type: "", frequency: { value: 0 }, Q: { value: 0 } }; } + createScriptProcessor() { const processor = { connect() {}, disconnect() {}, onaudioprocess: null }; processors.push(processor); return processor; } + createGain() { return { connect() {}, disconnect() {}, gain: { value: 1 } }; } + resume() { return Promise.resolve(); } + close() { this.state = "closed"; return Promise.resolve(); } + } + globalThis.window = { AudioContext }; + const ears = createSidecarWhisperEars({ + maxUtteranceMs: 99, + fetchImpl: async () => new Response(JSON.stringify({ ok: true, session: 1, kind: "final", text: "heard" })), + setTimeout: timers.setTimeout, + clearTimeout: timers.clearTimeout, + })({ onPartial() {}, onFinal() {}, onError() {} }, {}); + try { + ears.listen(); + const initialTimer = timers.pendingIds(99)[0]; + processors[0].onaudioprocess({ inputBuffer: { getChannelData: () => new Float32Array(4096).fill(0.1) } }); + const utteranceTimer = timers.pendingIds(99)[0]; + assert.notEqual(utteranceTimer, initialTimer, "speech replaces the initial-silence timer with a full utterance cap"); + } finally { + ears.close(); + globalThis.window = priorWindow; + } +}); + +test("pre-speech silence is not copied into Whisper capture", () => { + const priorWindow = globalThis.window; + const processors = []; + class AudioContext { + sampleRate = 48_000; + state = "running"; + destination = {}; + createMediaStreamSource() { return { connect() {}, disconnect() {} }; } + createBiquadFilter() { return { connect() {}, disconnect() {}, type: "", frequency: { value: 0 }, Q: { value: 0 } }; } + createScriptProcessor() { const processor = { connect() {}, disconnect() {}, onaudioprocess: null }; processors.push(processor); return processor; } + createGain() { return { connect() {}, disconnect() {}, gain: { value: 1 } }; } + resume() { return Promise.resolve(); } + close() { this.state = "closed"; return Promise.resolve(); } + } + globalThis.window = { AudioContext }; + const quiet = new Float32Array(4_096); + let copies = 0; + Object.defineProperty(quiet, "slice", { + value() { copies += 1; return new Float32Array(quiet); }, + }); + const ears = createSidecarWhisperEars()({ onPartial() {}, onFinal() {}, onError() {} }, {}); + try { + ears.listen(); + processors[0].onaudioprocess({ inputBuffer: { getChannelData: () => quiet } }); + assert.equal(copies, 0, "initial silence is discarded without allocating a PCM copy"); + } finally { + ears.close(); + globalThis.window = priorWindow; + } +}); + +test("voice activity drops leading silence before sending the final Whisper WAV", async () => { + const priorWindow = globalThis.window; + const timers = fakeTimers(); + const processors = []; + class AudioContext { + sampleRate = 48_000; + state = "running"; + destination = {}; + createMediaStreamSource() { return { connect() {}, disconnect() {} }; } + createBiquadFilter() { return { connect() {}, disconnect() {}, type: "", frequency: { value: 0 }, Q: { value: 0 } }; } + createScriptProcessor() { const processor = { connect() {}, disconnect() {}, onaudioprocess: null }; processors.push(processor); return processor; } + createGain() { return { connect() {}, disconnect() {}, gain: { value: 1 } }; } + resume() { return Promise.resolve(); } + close() { this.state = "closed"; return Promise.resolve(); } + } + globalThis.window = { AudioContext }; + let wavBytes = 0; + const ears = createSidecarWhisperEars({ + stabilityMs: 1, + maxUtteranceMs: 99, + fetchImpl: async (_url, init) => { + const audio = (init.body as FormData).get("audio") as Blob; + wavBytes = audio.size; + return new Response(JSON.stringify({ ok: true, session: 1, kind: "final", text: "heard" })); + }, + setTimeout: timers.setTimeout, + clearTimeout: timers.clearTimeout, + })({ onPartial() {}, onFinal() {}, onError() {} }, {}); + try { + ears.listen(); + processors[0].onaudioprocess({ inputBuffer: { getChannelData: () => new Float32Array(4_096) } }); + processors[0].onaudioprocess({ inputBuffer: { getChannelData: () => new Float32Array(4_096).fill(0.1) } }); + timers.fire(1); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal( + wavBytes, + 44 + 2 * Math.round(4_096 * WHISPER_SAMPLE_RATE / 48_000), + "the final WAV contains the voiced buffer but not the preceding silent buffer", + ); + } finally { + ears.close(); + globalThis.window = priorWindow; + } +}); diff --git a/src/lib/voice/sidecar-whisper.ts b/src/lib/voice/sidecar-whisper.ts new file mode 100644 index 000000000..89adcb137 --- /dev/null +++ b/src/lib/voice/sidecar-whisper.ts @@ -0,0 +1,169 @@ +// Server-side whisper.cpp runner for Cave's local sidecar. +// +// The browser only sends a bounded WAV utterance to this loopback route. The +// selected GGML model and whisper-cli process remain sidecar-owned, so neither +// model paths nor command arguments are client-controlled. + +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { + speechEnginesReadiness, + type SpeechEnginesReadiness, + type SpeechModelReadiness, +} from "./speech-models.ts"; + +const execFileAsync = promisify(execFile); + +export const MAX_WHISPER_WAV_BYTES = 10 * 1024 * 1024; +export const WHISPER_TIMEOUT_MS = 120_000; + +export class SidecarWhisperError extends Error { + readonly code: "whisper_model_not_ready" | "whisper_unavailable" | "whisper_failed" | "whisper_empty"; + readonly hint: string; + + constructor( + code: "whisper_model_not_ready" | "whisper_unavailable" | "whisper_failed" | "whisper_empty", + hint: string, + ) { + super(code); + this.name = "SidecarWhisperError"; + this.code = code; + this.hint = hint; + } +} + +export type ReadyWhisperModel = Pick; + +type ModelFileMetadata = { size: number; mtimeMs: number; isFile(): boolean }; + +/** Cache the verified selected model until its on-disk metadata changes. + * Readiness verifies complete model hashes, so repeating it for every partial + * decode needlessly streams hundreds of megabytes from disk. */ +export function createReadyWhisperModelCache( + loadEngines: () => Promise> = speechEnginesReadiness, + statFile: (filePath: string) => Promise = stat, +): () => Promise { + let cached: { model: ReadyWhisperModel; size: number; mtimeMs: number } | null = null; + let loading: Promise | null = null; + + return async () => { + if (cached) { + try { + const info = await statFile(cached.model.path); + if (info.isFile() && info.size === cached.size && info.mtimeMs === cached.mtimeMs) return cached.model; + } catch { + // A removed/unreadable model must be fully re-verified before use. + } + cached = null; + } + if (loading) return loading; + loading = (async () => { + const engines = await loadEngines(); + const candidate = engines.stt.find((model) => model.engine === "whisper" && model.ready); + if (!candidate) return null; + const model = { id: candidate.id, name: candidate.name, path: candidate.path }; + try { + const info = await statFile(model.path); + if (!info.isFile()) return null; + cached = { model, size: info.size, mtimeMs: info.mtimeMs }; + } catch { + return null; + } + return model; + })().finally(() => { loading = null; }); + return loading; + }; +} + +export const readyWhisperModel = createReadyWhisperModelCache(); + +export function whisperCliCommand(env: NodeJS.ProcessEnv = process.env): string { + return env.COVEN_WHISPER_CPP_BIN?.trim() || "whisper-cli"; +} + +/** Probe the executable separately from the downloaded GGML model. A model + * alone must never make the client advertise an engine that cannot start. */ +export async function whisperRuntimeAvailable(command = whisperCliCommand()): Promise { + try { + await execFileAsync(command, ["--version"], { + timeout: 1_500, + maxBuffer: 64 * 1024, + windowsHide: true, + }); + return true; + } catch { + return false; + } +} + +export function whisperCliArgs(modelPath: string, wavPath: string, outputStem: string, lang?: string): string[] { + return ["-m", modelPath, "-f", wavPath, "-otxt", "-of", outputStem, ...(lang ? ["-l", lang] : [])]; +} + +type RunWhisper = (command: string, args: string[], signal?: AbortSignal) => Promise; + +async function defaultRunWhisper(command: string, args: string[], signal?: AbortSignal): Promise { + try { + await execFileAsync(command, args, { + timeout: WHISPER_TIMEOUT_MS, + maxBuffer: 1024 * 1024, + windowsHide: true, + signal, + }); + } catch (error) { + if ((error as NodeJS.ErrnoException).name === "AbortError" || signal?.aborted) { + throw new SidecarWhisperError("whisper_failed", "Local Whisper transcription was cancelled."); + } + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT") { + throw new SidecarWhisperError( + "whisper_unavailable", + "The local Whisper runtime is unavailable. Install whisper.cpp or set COVEN_WHISPER_CPP_BIN to its whisper-cli executable.", + ); + } + throw new SidecarWhisperError( + "whisper_failed", + "Local Whisper could not transcribe that utterance. Check the downloaded model and try again.", + ); + } +} + +export async function transcribeSidecarWav( + wav: Uint8Array, + model: ReadyWhisperModel, + options: { lang?: string; run?: RunWhisper; tempRoot?: string; signal?: AbortSignal } = {}, +): Promise { + if (wav.byteLength === 0 || wav.byteLength > MAX_WHISPER_WAV_BYTES) { + throw new SidecarWhisperError("whisper_failed", "The recorded utterance is too large for local Whisper."); + } + const tempDir = await mkdtemp(path.join(options.tempRoot ?? os.tmpdir(), "coven-whisper-")); + const wavPath = path.join(tempDir, "utterance.wav"); + const outputStem = path.join(tempDir, "transcript"); + try { + await writeFile(wavPath, wav, { mode: 0o600 }); + if (options.signal?.aborted) { + throw new SidecarWhisperError("whisper_failed", "Local Whisper transcription was cancelled."); + } + await (options.run ?? defaultRunWhisper)( + whisperCliCommand(), + whisperCliArgs(model.path, wavPath, outputStem, options.lang), + options.signal, + ); + const text = (await readFile(`${outputStem}.txt`, "utf8")).trim(); + if (!text) { + throw new SidecarWhisperError("whisper_empty", "Local Whisper did not hear any words. Try speaking a little closer to the microphone."); + } + return text; + } catch (error) { + if (error instanceof SidecarWhisperError) throw error; + throw new SidecarWhisperError( + "whisper_failed", + "Local Whisper could not read its transcription result. Try again.", + ); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +} diff --git a/src/lib/voice/speech-loop.ts b/src/lib/voice/speech-loop.ts index d465e6a4a..de21e79e6 100644 --- a/src/lib/voice/speech-loop.ts +++ b/src/lib/voice/speech-loop.ts @@ -155,7 +155,9 @@ export type SpeechEars = { close(): void; }; -export type SpeechEarsFactory = (handlers: SpeechEarsHandlers) => SpeechEars; +/** The loop supplies its microphone stream so WebView ears can share the + * same permission and mute policy. Existing ears deliberately ignore it. */ +export type SpeechEarsFactory = (handlers: SpeechEarsHandlers, mic?: MediaStream) => SpeechEars; /** WebSpeech ears — SpeechRecognition where the WebView has it (Chromium * web builds). Returns null when this window has no engine. */ @@ -266,7 +268,7 @@ export function connectSpeechLoop(opts: SpeechLoopOptions): LiveSession { onError: (code, hint) => { if (!closed) callbacks.onError(new VoiceConnectError(code, hint)); }, - }); + }, mic); const listen = () => { if (closed || muted || speaking) return; diff --git a/src/lib/voice/types.ts b/src/lib/voice/types.ts index bb0f2a032..efcffb625 100644 --- a/src/lib/voice/types.ts +++ b/src/lib/voice/types.ts @@ -64,8 +64,14 @@ export interface LiveSession { * - "native-dictation": macOS SFSpeechRecognizer via Apple's dictation * service (no local model for the language). * - "web-speech": the browser's SpeechRecognition (Chromium's is a cloud - * service). */ -export type VoiceEarsEngine = "native-on-device" | "native-dictation" | "web-speech"; + * service). + * - "sidecar-whisper": a downloaded whisper.cpp model running in Cave's + * local sidecar; audio never leaves this device. */ +export type VoiceEarsEngine = + | "sidecar-whisper" + | "native-on-device" + | "native-dictation" + | "web-speech"; /** * Connection-phase error: `message` stays a stable machine code (e.g.