Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 34 additions & 12 deletions src/helpers/audioManager.js
Original file line number Diff line number Diff line change
Expand Up @@ -295,8 +295,13 @@ const STREAMING_FINAL_CEILING_MS = 2000;
// buildStreamingSessionOptions already stamps it) is pinned by
// audioManagerStreamingRouting.test.js: the hardened main-process allowlist
// fails closed on an options object that lost the tag (#1624).
const makeDictationRealtimeProvider = (id) => ({
//
// `cloudMetered` is per-caller rather than per-factory: these two share an IPC
// surface but not a billing relationship — OpenAI realtime runs as OpenWhispr
// Cloud's own upstream, Tinfoil never does.
const makeDictationRealtimeProvider = (id, { cloudMetered = false } = {}) => ({
awaitsFinalTranscript: true,
cloudMetered,
warmup: (opts) => window.electronAPI.dictationRealtimeWarmup({ ...opts, provider: id }),
start: (opts) => window.electronAPI.dictationRealtimeStart({ ...opts, provider: id }),
send: (buf) => window.electronAPI.dictationRealtimeSend(buf),
Expand All @@ -307,8 +312,16 @@ const makeDictationRealtimeProvider = (id) => ({
onSessionEnd: (cb) => window.electronAPI.onDictationRealtimeSessionEnd(cb),
});

// `cloudMetered` marks the providers whose audio travels through OpenWhispr
// Cloud, and therefore the only sessions allowed to report usage back to it.
// That report carries `sendLogs` — the transcript itself — so a provider that
// transcribes elsewhere (the user's own key, their own server) must stay silent
// or the feature ships their text to the service they deliberately routed
// around. The flag is opt-in: a backend added later never reports until someone
// states otherwise, so an oversight costs a usage row rather than a transcript.
const STREAMING_PROVIDERS = {
deepgram: {
cloudMetered: true,
warmup: (opts) => window.electronAPI.deepgramStreamingWarmup(opts),
start: (opts) => window.electronAPI.deepgramStreamingStart(opts),
send: (buf) => window.electronAPI.deepgramStreamingSend(buf),
Expand All @@ -321,6 +334,7 @@ const STREAMING_PROVIDERS = {
onSessionEnd: (cb) => window.electronAPI.onDeepgramSessionEnd(cb),
},
assemblyai: {
cloudMetered: true,
warmup: (opts) => window.electronAPI.assemblyAiStreamingWarmup(opts),
start: (opts) => window.electronAPI.assemblyAiStreamingStart(opts),
send: (buf) => window.electronAPI.assemblyAiStreamingSend(buf),
Expand All @@ -332,7 +346,10 @@ const STREAMING_PROVIDERS = {
onError: (cb) => window.electronAPI.onAssemblyAiError(cb),
onSessionEnd: (cb) => window.electronAPI.onAssemblyAiSessionEnd(cb),
},
"openai-realtime": makeDictationRealtimeProvider("openai-realtime"),
// Runs both as OpenWhispr Cloud's own upstream and, in BYOK mode, against the
// user's key — so the mode decides whether a session reports.
"openai-realtime": makeDictationRealtimeProvider("openai-realtime", { cloudMetered: true }),
// Streams over Corti's own WSS on the user's credentials.
corti: {
warmup: (opts) => window.electronAPI.cortiStreamingWarmup(opts),
start: (opts) => window.electronAPI.cortiStreamingStart(opts),
Expand All @@ -345,6 +362,7 @@ const STREAMING_PROVIDERS = {
onError: (cb) => window.electronAPI.onCortiError(cb),
onSessionEnd: (cb) => window.electronAPI.onCortiSessionEnd(cb),
},
// Streams against the user's own Tinfoil key.
"tinfoil-realtime": makeDictationRealtimeProvider("tinfoil-realtime"),
};

Expand Down Expand Up @@ -478,6 +496,7 @@ class AudioManager {
this.streamingPartialText = "";
this.streamingTextBump = null;
this.streamingTextDebounce = null;
this.streamingSessionMetered = false;
this.cachedMicDeviceId = null;
this.rejectedMicDeviceId = null;
this.persistentAudioContext = null;
Expand Down Expand Up @@ -4166,15 +4185,18 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
const result = await withSessionRefresh(async () => {
const streamingSettings = getSettings();
const { useLocalWhisper } = streamingSettings;
const res = await provider.start(
buildStreamingSessionOptions({
providerName: this.getStreamingProviderName(),
settings: streamingSettings,
language: this.getEffectiveSttLanguage(streamingSettings),
keyterms: this.getKeyterms(),
voiceAgentRequested: this.voiceAgentRequested,
})
);
const sessionOptions = buildStreamingSessionOptions({
providerName: this.getStreamingProviderName(),
settings: streamingSettings,
language: this.getEffectiveSttLanguage(streamingSettings),
keyterms: this.getKeyterms(),
voiceAgentRequested: this.voiceAgentRequested,
});
// Pin the usage decision to the routing this session actually connects
// with; settings may change before it ends, the routing will not.
this.streamingSessionMetered =
!!provider.cloudMetered && sessionOptions.mode === "openwhispr";
const res = await provider.start(sessionOptions);

if (!res.success) {
if (res.code === "NO_API") {
Expand Down Expand Up @@ -4843,7 +4865,7 @@ registerProcessor("pcm-streaming-processor", PCMStreamingProcessor);
...(batchWarning ? { warning: batchWarning } : {}),
});

if (!usedBatchFallback) {
if (!usedBatchFallback && this.streamingSessionMetered) {
(async () => {
try {
await withSessionRefresh(async () => {
Expand Down
158 changes: 158 additions & 0 deletions test/helpers/audioManagerUsageGate.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const { createRendererServer, installBrowserGlobals } = require("../lib/rendererTestHarness");

// The streaming usage report carries `sendLogs` — the transcript text — so it
// may only fire for a session whose audio actually travelled through OpenWhispr
// Cloud. The gate in startStreamingRecording reads two things:
//
// this.streamingSessionMetered =
// !!provider.cloudMetered && sessionOptions.mode === "openwhispr";
//
// Both come from production code reached here: `provider` from the real
// STREAMING_PROVIDERS registry via getStreamingProvider(), and `mode` from the
// real buildStreamingSessionOptions(). Each case below drives the real
// resolveStreamingProviderName() from a settings state, so the table also pins
// which provider a given configuration streams through.
//
// What this cannot reach: the gate expression itself, and its one consumer
// (`if (!usedBatchFallback && this.streamingSessionMetered)`), both of which sit
// inside startStreamingRecording after the mic/worklet setup. This pins the
// inputs — a backend added without deciding, or a BYOK provider flipped to
// metered, fails here.

async function loadRoutingSurface(t) {
installBrowserGlobals(t);
const vite = await createRendererServer(t, {
cachePrefix: "openwhispr-usage-gate-test-",
mockModules: {
"/utils/logger":
"export default { debug() {}, info() {}, warn() {}, error() {}, logReasoning() {} };",
// Settings are read through a global so one loaded module can serve every
// case; the real getSettings() is what production calls here too.
"/stores/settingsStore": `
export const getSettings = () => globalThis.__owUsageGateSettings ?? {};
export const getEffectiveCleanupModel = () => null;
export const isCloudCleanupMode = () => false;
export const isCloudDictationAgentMode = () => false;
export const isCloudTranslationMode = () => false;
`,
"/services/ReasoningService": "export default { processText: async (t) => t };",
"/services/SyncService.js": "export const syncService = {};",
"/lib/auth": "export const withSessionRefresh = (fn) => fn();",
"/utils/permissions": "export const isAccessibilitySkipped = () => false;",
},
});

const AudioManager = (await vite.ssrLoadModule("/helpers/audioManager.js")).default;
const { buildStreamingSessionOptions } = await vite.ssrLoadModule(
"/helpers/dictationStreamingRouting.js"
);

t.after(() => {
delete globalThis.__owUsageGateSettings;
});

return { AudioManager, buildStreamingSessionOptions };
}

// Resolves a settings state the way startStreamingRecording does: real
// resolver, real registry, real options builder.
function resolveSession({ AudioManager, buildStreamingSessionOptions }, { settings, sttConfig }) {
globalThis.__owUsageGateSettings = settings;

const manager = Object.create(AudioManager.prototype);
manager.context = "dictation";
manager.sttConfig = sttConfig ?? null;

const providerName = manager.getStreamingProviderName();
return {
providerName,
provider: manager.getStreamingProvider(),
options: buildStreamingSessionOptions({
providerName,
settings,
language: "en",
keyterms: [],
}),
};
}

const CASES = [
{
name: "OpenWhispr-managed realtime dictation",
settings: {
cloudTranscriptionModel: "gpt-4o-mini-transcribe",
cloudTranscriptionMode: "openwhispr",
},
providerName: "openai-realtime",
cloudMetered: true,
mode: "openwhispr",
reportsUsage: true,
},
{
name: "the same provider on the user's own OpenAI key",
settings: { cloudTranscriptionModel: "gpt-4o-mini-transcribe", cloudTranscriptionMode: "byok" },
providerName: "openai-realtime",
cloudMetered: true,
mode: "byok",
reportsUsage: false,
},
{
name: "Corti on the user's own credentials",
settings: { cloudTranscriptionProvider: "corti", cloudTranscriptionMode: "byok" },
providerName: "corti",
cloudMetered: false,
mode: "byok",
reportsUsage: false,
},
{
name: "Tinfoil realtime on the user's own key",
settings: { cloudTranscriptionProvider: "tinfoil", cloudTranscriptionMode: "byok" },
providerName: "tinfoil-realtime",
cloudMetered: false,
mode: "byok",
reportsUsage: false,
},
{
name: "Deepgram through OpenWhispr Cloud",
settings: { cloudTranscriptionMode: "openwhispr" },
sttConfig: { streamingProvider: "deepgram" },
providerName: "deepgram",
cloudMetered: true,
mode: "openwhispr",
reportsUsage: true,
},
{
name: "AssemblyAI through OpenWhispr Cloud",
settings: { cloudTranscriptionMode: "openwhispr" },
sttConfig: { streamingProvider: "assemblyai" },
providerName: "assemblyai",
cloudMetered: true,
mode: "openwhispr",
reportsUsage: true,
},
];

test("usage reporting is gated to OpenWhispr-cloud transcriptions", async (t) => {
const surface = await loadRoutingSurface(t);

for (const testCase of CASES) {
const { providerName, provider, options } = resolveSession(surface, testCase);

assert.equal(providerName, testCase.providerName, `${testCase.name}: routed provider`);
// Coerced because both spellings of "not metered" are legitimate: the
// dictation-realtime factory passes an explicit false, while a provider that
// never opts in simply omits the key. The gate reads it the same way.
assert.equal(
!!provider.cloudMetered,
testCase.cloudMetered,
`${testCase.name}: cloudMetered declaration`
);
assert.equal(options.mode, testCase.mode, `${testCase.name}: mode sent at connect time`);

// Mirrors the gate in startStreamingRecording.
const metered = !!provider.cloudMetered && options.mode === "openwhispr";
assert.equal(metered, testCase.reportsUsage, `${testCase.name}: reports usage`);
}
});
Loading