${escapeHtml(t('tr.event.llm_response'))}${stepBadge}${usage}${lat}${ts}
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 4c32ba97b..0dd252e54 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -95,6 +95,7 @@ import {
} from '../providers/provider-compatibility.js';
import { extractFirstJsonObject } from './json-extract.js';
import { repairAssistantDisplayText, sanitizeText as sanitizePlannerText } from './text-sanitize.js';
+import { emptyOutputFailureMessage, modelOutputDiagnostics } from './model-output-diagnostics.js';
import { buildCustomSkillsPrompt, buildSkillLoaderDefinition, buildSkillToolDefinitions, buildSkillToolRegistry, getEligibleCustomSkills, getEligibleSkillCatalog, normalizeCustomSkills } from './skills.js';
import { publicMediaUrlNeedsExplicitTarget } from './public-media-url.js';
import { USER_MEMORY_DEFAULT_MAX_PROMPT_CHARS, formatUserMemoryPrompt, normalizeUserMemoryMaxPromptChars, normalizeUserMemoryStore } from './user-memory.js';
@@ -21274,6 +21275,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
model: provider.model,
messageCount: prunedMessages.length,
toolsCount: (chatOpts.tools || []).length,
+ requestedMaxTokens: chatOpts.maxTokens,
...Agent._traceMediaCounts(prunedMessages),
}, {
messages: prunedMessages,
@@ -21299,6 +21301,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
usage: result.usage,
latencyMs: Date.now() - _llmStart,
model: provider.model,
+ ...modelOutputDiagnostics(result, {
+ requestedMaxTokens: chatOpts.maxTokens,
+ recoveryAttempt: emptyOutputRecoveryAttempted ? 2 : 1,
+ }),
});
try {
if (shouldOrderInteractiveAskTrace) await queueAskStreamingTraceWrite(writeResponseTrace);
@@ -21546,7 +21552,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this._persist(tabId);
break;
}
- finalResponse = '[Agent emitted no output and no tool call, even after a recovery nudge. This usually means the task exceeded the current model\'s capability or context budget. Try a stronger model, raise the step limit in settings, or break the task into smaller parts.]';
+ finalResponse = emptyOutputFailureMessage(modelOutputDiagnostics(result, {
+ requestedMaxTokens: 4096,
+ recoveryAttempt: 2,
+ }));
_traceStatus = 'empty_output';
traceFailureCode = 'EMPTY_RESPONSE';
messages.push({ role: 'assistant', content: finalResponse });
@@ -22033,6 +22042,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
let hasToolCalls = false;
let responseItems = null;
let reasoningContent = '';
+ let streamUsage = null;
+ let finishReason = '';
const streamOpts = this._cloudGenerationOptions(provider, {
tools: provider.supportsTools && tools.length > 0 ? tools : undefined,
@@ -22053,6 +22064,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
await closeTraceStep({ ok: false, code: 'COST_LIMIT' });
return finish(beforeCost, 'cost_limit');
}
+ if (runId) {
+ await trace.recordLLMRequest(runId, steps, {
+ providerClass: provider.constructor.name,
+ model: provider.model,
+ messageCount: prunedMessages.length,
+ toolsCount: (streamOpts.tools || []).length,
+ requestedMaxTokens: streamOpts.maxTokens,
+ ...Agent._traceMediaCounts(prunedMessages),
+ }, {
+ messages: prunedMessages,
+ tools: streamOpts.tools || [],
+ runtimeMode: mode,
+ });
+ }
+ const _llmStart = Date.now();
let costStopMessage = '';
for await (const chunk of provider.chatStream(prunedMessages, streamOpts)) {
@@ -22063,6 +22089,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
} else if (chunk.type === 'reasoning') {
reasoningContent += String(chunk.content || '');
} else if (chunk.type === 'usage') {
+ streamUsage = chunk.usage || streamUsage;
costStopMessage = (await this._recordCostUsage(provider, chunk.usage, costState)) || costStopMessage;
} else if (chunk.type === 'tool_call') {
streamEmittedOutput = true;
@@ -22094,14 +22121,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (Array.isArray(chunk.responseItems) && chunk.responseItems.length) {
responseItems = chunk.responseItems;
}
+ finishReason = String(
+ chunk.finishReason
+ ?? chunk.finish_reason
+ ?? chunk.stopReason
+ ?? chunk.stop_reason
+ ?? '',
+ );
+ if (chunk.usage && !streamUsage) {
+ streamUsage = chunk.usage;
+ costStopMessage = (await this._recordCostUsage(provider, chunk.usage, costState)) || costStopMessage;
+ }
break;
}
}
fullText = Agent._stripReasoningTags(fullText);
+ const streamedToolCalls = hasToolCalls ? Object.values(toolCallsAccumulator) : [];
+ const outputDiagnostics = modelOutputDiagnostics({
+ content: fullText,
+ toolCalls: streamedToolCalls,
+ reasoningContent,
+ usage: streamUsage,
+ finishReason,
+ responseItems,
+ }, {
+ requestedMaxTokens: streamOpts.maxTokens,
+ recoveryAttempt: emptyOutputRecoveryAttempted ? 2 : 1,
+ });
+ if (runId) {
+ await trace.recordLLMResponse(runId, steps, {
+ content: fullText,
+ toolCalls: streamedToolCalls,
+ usage: streamUsage,
+ latencyMs: Date.now() - _llmStart,
+ model: provider.model,
+ ...outputDiagnostics,
+ });
+ }
closeTraceStep(this._traceStepEndForResult({
content: fullText,
- toolCalls: hasToolCalls ? Object.values(toolCallsAccumulator) : [],
+ toolCalls: streamedToolCalls,
}));
// Fallback: parse tool calls from streamed text if structured calls are missing.
@@ -22198,7 +22258,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this._persist(tabId);
return finish(scheduledResume.message, 'scheduled_resume');
}
- const failMsg = '[Agent emitted no output and no tool call, even after a recovery nudge. This usually means the task exceeded the current model\'s capability or context budget. Try a stronger model, raise the step limit in settings, or break the task into smaller parts.]';
+ const failMsg = emptyOutputFailureMessage(outputDiagnostics);
messages.push({ role: 'assistant', content: failMsg });
onUpdate('warning', { message: failMsg });
this._persist(tabId);
diff --git a/src/firefox/src/agent/model-output-diagnostics.js b/src/firefox/src/agent/model-output-diagnostics.js
new file mode 100644
index 000000000..47ac13d58
--- /dev/null
+++ b/src/firefox/src/agent/model-output-diagnostics.js
@@ -0,0 +1,115 @@
+const OUTPUT_LIMIT_RE = /(?:^|[_\s-])(?:length|max(?:imum)?(?:[_\s-]*(?:output|completion))?[_\s-]*tokens?|output[_\s-]*limit|token[_\s-]*limit)(?:$|[_\s-])/i;
+const CONTENT_FILTER_RE = /content[_\s-]*filter|safety|blocked|refusal/i;
+
+function finiteNonNegative(...values) {
+ for (const value of values) {
+ if (value == null || value === '') continue;
+ const number = Number(value);
+ if (Number.isFinite(number) && number >= 0) return Math.floor(number);
+ }
+ return null;
+}
+
+function finitePositive(value) {
+ const number = Number(value);
+ return Number.isFinite(number) && number > 0 ? Math.floor(number) : null;
+}
+
+function finishReasonFor(result) {
+ const raw = result?.raw || {};
+ return String(
+ result?.finishReason
+ ?? result?.finish_reason
+ ?? result?.stopReason
+ ?? result?.stop_reason
+ ?? raw?.choices?.[0]?.finish_reason
+ ?? raw?.finish_reason
+ ?? raw?.stopReason
+ ?? raw?.stop_reason
+ ?? '',
+ ).replace(/[\u0000-\u001f\u007f]/g, '').trim().slice(0, 80);
+}
+
+function usageFor(result) {
+ return result?.usage && typeof result.usage === 'object'
+ ? result.usage
+ : (result?.raw?.usage && typeof result.raw.usage === 'object' ? result.raw.usage : {});
+}
+
+function responseHasReasoningItem(result) {
+ const items = Array.isArray(result?.responseItems)
+ ? result.responseItems
+ : (Array.isArray(result?.raw?.output) ? result.raw.output : []);
+ return items.some((item) => {
+ if (item?.type === 'reasoning') return true;
+ if (item?.type !== 'webbrain_provider_replay' || !Array.isArray(item.content)) return false;
+ return item.content.some(block => block?.type === 'thinking' || block?.type === 'redacted_thinking');
+ });
+}
+
+export function modelOutputDiagnostics(result, { requestedMaxTokens = null, recoveryAttempt = 0 } = {}) {
+ const contentChars = typeof result?.content === 'string' ? result.content.trim().length : 0;
+ const toolCallCount = Array.isArray(result?.toolCalls) ? result.toolCalls.length : 0;
+ const reasoningChars = typeof result?.reasoningContent === 'string' ? result.reasoningContent.length : 0;
+ const usage = usageFor(result);
+ const reasoningTokens = finiteNonNegative(
+ usage?.completion_tokens_details?.reasoning_tokens,
+ usage?.output_tokens_details?.reasoning_tokens,
+ usage?.reasoning_tokens,
+ );
+ const outputTokens = finiteNonNegative(
+ usage?.completion_tokens,
+ usage?.output_tokens,
+ usage?.completionTokens,
+ usage?.outputTokens,
+ );
+ const normalizedMaxTokens = finitePositive(requestedMaxTokens);
+ const finishReason = finishReasonFor(result);
+ const reasoningPresent = reasoningChars > 0
+ || (reasoningTokens != null && reasoningTokens > 0)
+ || responseHasReasoningItem(result);
+ const empty = contentChars === 0 && toolCallCount === 0;
+ let emptyReason = null;
+
+ if (empty) {
+ if (CONTENT_FILTER_RE.test(finishReason)) {
+ emptyReason = 'content_filter';
+ } else if (
+ OUTPUT_LIMIT_RE.test(` ${finishReason} `)
+ || (normalizedMaxTokens != null && outputTokens != null && outputTokens >= normalizedMaxTokens)
+ ) {
+ emptyReason = 'output_limit';
+ } else if (reasoningPresent) {
+ emptyReason = 'reasoning_only';
+ } else {
+ emptyReason = 'provider_empty';
+ }
+ }
+
+ return {
+ empty,
+ emptyReason,
+ finishReason: finishReason || null,
+ contentChars,
+ toolCallCount,
+ reasoningPresent,
+ reasoningChars,
+ reasoningTokens,
+ outputTokens,
+ requestedMaxTokens: normalizedMaxTokens,
+ recoveryAttempt: Number.isInteger(recoveryAttempt) && recoveryAttempt > 0 ? recoveryAttempt : 0,
+ };
+}
+
+export function emptyOutputFailureMessage(diagnostics = {}) {
+ switch (diagnostics.emptyReason) {
+ case 'output_limit':
+ return '[The model reached its configured response-token limit without producing visible text or a tool call, even after a recovery nudge. Reduce reasoning effort or choose another model/provider.]';
+ case 'reasoning_only':
+ return '[The latest model response contained reasoning but no visible answer or tool call after the earlier empty response. Reduce reasoning effort or choose another model/provider.]';
+ case 'content_filter':
+ return '[The provider filtered the latest response after an earlier empty response, so no visible text or tool call was returned. Revise the request or try another provider.]';
+ default:
+ return '[The provider returned an empty completion with no visible text or tool call twice. Retry later or choose another model/provider.]';
+ }
+}
diff --git a/src/firefox/src/agent/trace-export.js b/src/firefox/src/agent/trace-export.js
index c738cbd50..a5b6c8a62 100644
--- a/src/firefox/src/agent/trace-export.js
+++ b/src/firefox/src/agent/trace-export.js
@@ -103,6 +103,24 @@ function renderLosslessRequest(messages, tools) {
function oneLine(t) { return String(t ?? '').replace(/\s+/g, ' ').trim(); }
function humanSize(n) { return n >= 1024 ? `${(n / 1024).toFixed(1)}kb` : `${n}b`; }
+function renderEmptyModelResponse(data) {
+ const details = [
+ `reason=${oneLine(data.emptyReason || 'unknown')}`,
+ data.finishReason ? `finish=${oneLine(data.finishReason)}` : '',
+ Number.isInteger(data.outputTokens) ? `output=${data.outputTokens} tokens` : '',
+ Number.isInteger(data.reasoningTokens)
+ ? `reasoning=${data.reasoningTokens} tokens`
+ : (Number.isInteger(data.reasoningChars) && data.reasoningChars > 0
+ ? `reasoning=${data.reasoningChars} chars`
+ : (data.reasoningPresent === true ? 'reasoning=present' : '')),
+ Number.isInteger(data.requestedMaxTokens) ? `limit=${data.requestedMaxTokens} tokens` : '',
+ Number.isInteger(data.recoveryAttempt) ? `attempt=${data.recoveryAttempt}` : '',
+ `${Number.isInteger(data.contentChars) ? data.contentChars : 0} visible chars`,
+ `${Number.isInteger(data.toolCallCount) ? data.toolCallCount : 0} tool calls`,
+ ].filter(Boolean).join(' 路 ');
+ return `- 馃 Empty model response: ${details}\n`;
+}
+
function truncate(text, limit) {
const s = String(text ?? '');
if (s.length <= limit) return s;
@@ -280,7 +298,12 @@ export function tracesToMarkdown(runsWithEvents, {
md += `- 馃 Model request: ${Number(d.messageCount) || 0} messages 路 ${Number(d.toolsCount) || 0} tools${media ? ` 路 ${media}` : ''}${renderLocalWikipediaRag(d.localWikipediaRag)}${renderPromptProvenance(d.promptProvenance)}${d.lossless === true ? renderLosslessRequest(d.messages, d.tools) : ''}\n`;
} else if (ev.kind === 'llm_response') {
const content = String(d.content || '').trim();
- if (!content) continue;
+ if (!content) {
+ if (!Array.isArray(d.toolCalls) || d.toolCalls.length === 0) {
+ md += renderEmptyModelResponse(d);
+ }
+ continue;
+ }
// Plan-before-Act runs record the planner call with phase:'planner'; keep
// it (derails often start in the plan) but label it and preserve its shape.
if (d.phase === 'planner') {
diff --git a/src/firefox/src/providers/openai.js b/src/firefox/src/providers/openai.js
index e47d77ccc..b23af0f03 100644
--- a/src/firefox/src/providers/openai.js
+++ b/src/firefox/src/providers/openai.js
@@ -978,6 +978,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
reasoningContent: message?.reasoning_content || message?.reasoning || '',
toolCalls: message?.tool_calls || null,
usage: data.usage || null,
+ finishReason: String(data?.choices?.[0]?.finish_reason || ''),
raw: data,
};
}
diff --git a/src/firefox/src/trace/recorder.js b/src/firefox/src/trace/recorder.js
index 272057f52..e5ac0a51a 100644
--- a/src/firefox/src/trace/recorder.js
+++ b/src/firefox/src/trace/recorder.js
@@ -648,7 +648,27 @@ export function recordLLMRequest(runId, step, payload, provenanceInput = null) {
});
}
-export function recordLLMResponse(runId, step, { content, toolCalls, usage, latencyMs, model, phase, attempt, repair }) {
+export function recordLLMResponse(runId, step, {
+ content,
+ toolCalls,
+ usage,
+ latencyMs,
+ model,
+ phase,
+ attempt,
+ repair,
+ empty,
+ emptyReason,
+ finishReason,
+ contentChars,
+ toolCallCount,
+ reasoningPresent,
+ reasoningChars,
+ reasoningTokens,
+ outputTokens,
+ requestedMaxTokens,
+ recoveryAttempt,
+}) {
return _appendEvent(runId, 'llm_response', {
step,
content: content || null,
@@ -665,6 +685,17 @@ export function recordLLMResponse(runId, step, { content, toolCalls, usage, late
...(phase ? { phase } : {}),
...(Number.isInteger(attempt) ? { attempt } : {}),
...(repair === true ? { repair: true } : {}),
+ ...(typeof empty === 'boolean' ? { empty } : {}),
+ ...(emptyReason ? { emptyReason } : {}),
+ ...(finishReason ? { finishReason } : {}),
+ ...(Number.isInteger(contentChars) ? { contentChars } : {}),
+ ...(Number.isInteger(toolCallCount) ? { toolCallCount } : {}),
+ ...(typeof reasoningPresent === 'boolean' ? { reasoningPresent } : {}),
+ ...(Number.isInteger(reasoningChars) ? { reasoningChars } : {}),
+ ...(Number.isInteger(reasoningTokens) ? { reasoningTokens } : {}),
+ ...(Number.isInteger(outputTokens) ? { outputTokens } : {}),
+ ...(Number.isInteger(requestedMaxTokens) ? { requestedMaxTokens } : {}),
+ ...(Number.isInteger(recoveryAttempt) && recoveryAttempt > 0 ? { recoveryAttempt } : {}),
});
}
diff --git a/src/firefox/src/ui/traces.js b/src/firefox/src/ui/traces.js
index f2aaa4d1c..d98c42ea0 100644
--- a/src/firefox/src/ui/traces.js
+++ b/src/firefox/src/ui/traces.js
@@ -525,9 +525,10 @@ function renderEvent(ev, shotCache, compact, objectUrls = new Set()) {
const usage = u ? `
${(u.prompt_tokens || 0).toLocaleString()} in / ${(u.completion_tokens || 0).toLocaleString()} out` : '';
const lat = ev.data?.latencyMs != null ? `
${ev.data.latencyMs} ms` : '';
const content = ev.data?.content;
+ const hasVisibleContent = typeof content === 'string' ? content.trim().length > 0 : !!content;
const toolCalls = ev.data?.toolCalls || [];
let body = '';
- if (content) {
+ if (hasVisibleContent) {
body += `
${escapeHtml(content)}
`;
}
if (toolCalls.length > 0) {
@@ -538,6 +539,21 @@ function renderEvent(ev, shotCache, compact, objectUrls = new Set()) {
}).join('');
body += `
${tcList}
`;
}
+ if (!hasVisibleContent && toolCalls.length === 0) {
+ const emptyDetails = [
+ `reason=${ev.data?.emptyReason || 'unknown'}`,
+ ev.data?.finishReason ? `finish=${ev.data.finishReason}` : '',
+ Number.isInteger(ev.data?.outputTokens) ? `output_tokens=${ev.data.outputTokens}` : '',
+ Number.isInteger(ev.data?.reasoningTokens)
+ ? `reasoning_tokens=${ev.data.reasoningTokens}`
+ : (ev.data?.reasoningPresent === true ? 'reasoning_present=true' : ''),
+ Number.isInteger(ev.data?.requestedMaxTokens) ? `requested_max_tokens=${ev.data.requestedMaxTokens}` : '',
+ Number.isInteger(ev.data?.recoveryAttempt) ? `attempt=${ev.data.recoveryAttempt}` : '',
+ `content_chars=${Number.isInteger(ev.data?.contentChars) ? ev.data.contentChars : 0}`,
+ `tool_calls=${Number.isInteger(ev.data?.toolCallCount) ? ev.data.toolCallCount : 0}`,
+ ].filter(Boolean).join(' 路 ');
+ body = `
EMPTY_RESPONSE 路 ${escapeHtml(emptyDetails)}
`;
+ }
return `
${escapeHtml(t('tr.event.llm_response'))}${stepBadge}${usage}${lat}${ts}
diff --git a/test/run.js b/test/run.js
index 191842037..6a5437032 100644
--- a/test/run.js
+++ b/test/run.js
@@ -292,6 +292,12 @@ const { tracesToMarkdown, sanitizeTraceExport } = await import(
const { tracesToMarkdown: tracesToMarkdownFx, sanitizeTraceExport: sanitizeTraceExportFx } = await import(
'file://' + path.join(ROOT, 'src/firefox/src/agent/trace-export.js').replace(/\\/g, '/')
);
+const ModelOutputDiagnosticsCh = await import(
+ 'file://' + path.join(ROOT, 'src/chrome/src/agent/model-output-diagnostics.js').replace(/\\/g, '/')
+);
+const ModelOutputDiagnosticsFx = await import(
+ 'file://' + path.join(ROOT, 'src/firefox/src/agent/model-output-diagnostics.js').replace(/\\/g, '/')
+);
const Utf8BudgetCh = await import(
'file://' + path.join(ROOT, 'src/chrome/src/trace/utf8-budget.js').replace(/\\/g, '/')
);
@@ -9794,6 +9800,9 @@ test('trace UI: renders the trajectory rows before the detailed event timeline',
assert.match(traces, /getSessionStats\(run\.conversationId\)/, `${browser}: conversation panel does not read indexed session stats`);
assert.match(traces, /trajectory-table/, `${browser}: trajectory table class missing from renderer`);
assert.match(traces, /renderStepTrajectory\(events, compact\)/, `${browser}: run view does not render the trajectory before details`);
+ assert.match(traces, /EMPTY_RESPONSE 路 \$\{escapeHtml\(emptyDetails\)\}/, `${browser}: empty model responses have no visible diagnostics`);
+ assert.match(traces, /const hasVisibleContent = typeof content === 'string' \? content\.trim\(\)\.length > 0 : !!content/, `${browser}: whitespace-only responses are not rendered as empty`);
+ assert.match(traces, /emptyReason \|\| 'unknown'/, `${browser}: legacy empty responses should not receive an invented cause`);
assert.match(html, /\.trajectory-table|\.trajectory-row/, `${browser}: trajectory table styles missing`);
assert.match(html, /\.conv-summary/, `${browser}: conversation statistics style missing`);
}
@@ -9823,6 +9832,134 @@ test('trace UI: renders collapsed session lineage groups with bounded-result war
}
});
+test('model output diagnostics: classifies empty completions without retaining reasoning text', () => {
+ for (const [label, diagnostics] of [
+ ['chrome', ModelOutputDiagnosticsCh],
+ ['firefox', ModelOutputDiagnosticsFx],
+ ]) {
+ const limited = diagnostics.modelOutputDiagnostics({
+ content: '',
+ toolCalls: [],
+ reasoningContent: 'private reasoning text',
+ usage: {
+ completion_tokens: 4096,
+ completion_tokens_details: { reasoning_tokens: 4096 },
+ },
+ finishReason: 'length',
+ }, { requestedMaxTokens: 4096, recoveryAttempt: 2 });
+ assert.deepEqual(limited, {
+ empty: true,
+ emptyReason: 'output_limit',
+ finishReason: 'length',
+ contentChars: 0,
+ toolCallCount: 0,
+ reasoningPresent: true,
+ reasoningChars: 22,
+ reasoningTokens: 4096,
+ outputTokens: 4096,
+ requestedMaxTokens: 4096,
+ recoveryAttempt: 2,
+ }, `${label}: output-limit diagnostics changed`);
+ assert.doesNotMatch(JSON.stringify(limited), /private reasoning text/, `${label}: reasoning text leaked into diagnostics`);
+ assert.match(diagnostics.emptyOutputFailureMessage(limited), /response-token limit/i, `${label}: output-limit guidance missing`);
+ assert.doesNotMatch(diagnostics.emptyOutputFailureMessage(limited), /step limit/i, `${label}: model output failures must not blame the agent step limit`);
+ assert.doesNotMatch(diagnostics.emptyOutputFailureMessage(limited), /increase the response-token limit/i, `${label}: output-limit guidance must not recommend an unavailable setting`);
+
+ assert.equal(diagnostics.modelOutputDiagnostics({
+ content: '',
+ toolCalls: null,
+ reasoningContent: 'thought',
+ usage: { completion_tokens: 12 },
+ }, { requestedMaxTokens: 4096 }).emptyReason, 'reasoning_only', `${label}: reasoning-only response not classified`);
+ assert.equal(diagnostics.modelOutputDiagnostics({
+ content: '',
+ toolCalls: [],
+ responseItems: [{ type: 'reasoning', encrypted_content: 'opaque' }],
+ }).emptyReason, 'reasoning_only', `${label}: encrypted reasoning item not detected without exposing it`);
+ assert.equal(diagnostics.modelOutputDiagnostics({
+ content: '',
+ toolCalls: [],
+ usage: { completion_tokens: 42 },
+ responseItems: [{
+ type: 'webbrain_provider_replay',
+ content: [{ type: 'redacted_thinking', data: 'opaque' }],
+ }],
+ }, { requestedMaxTokens: 4096 }).emptyReason, 'reasoning_only', `${label}: provider replay reasoning envelope not detected`);
+ assert.equal(diagnostics.modelOutputDiagnostics({
+ content: '',
+ toolCalls: [],
+ finishReason: 'content_filter',
+ }).emptyReason, 'content_filter', `${label}: filtered response not classified`);
+ assert.equal(diagnostics.modelOutputDiagnostics({
+ content: '',
+ toolCalls: [],
+ usage: { completion_tokens: 0 },
+ }, { requestedMaxTokens: 4096 }).emptyReason, 'provider_empty', `${label}: unexplained empty response not classified`);
+ assert.equal(diagnostics.modelOutputDiagnostics({
+ content: '',
+ toolCalls: [],
+ raw: { choices: [{ finish_reason: 'max_tokens' }] },
+ }).emptyReason, 'output_limit', `${label}: raw provider finish reason not classified`);
+ assert.equal(diagnostics.modelOutputDiagnostics({
+ content: 'done',
+ toolCalls: [],
+ reasoningContent: 'thought',
+ }).emptyReason, null, `${label}: visible output must not receive an empty classification`);
+ assert.equal(diagnostics.modelOutputDiagnostics({
+ content: '',
+ toolCalls: [{ id: 'call_1' }],
+ }).empty, false, `${label}: tool output must not be empty`);
+ const reasoningMessage = diagnostics.emptyOutputFailureMessage({ emptyReason: 'reasoning_only' });
+ assert.match(reasoningMessage, /latest model response/i, `${label}: reasoning-only guidance should identify the observed attempt`);
+ assert.doesNotMatch(reasoningMessage, /twice/i, `${label}: reasoning-only guidance must not invent the first attempt's cause`);
+ const filteredMessage = diagnostics.emptyOutputFailureMessage({ emptyReason: 'content_filter' });
+ assert.match(filteredMessage, /latest response/i, `${label}: filter guidance should identify the observed attempt`);
+ assert.doesNotMatch(filteredMessage, /both attempts/i, `${label}: filter guidance must not invent the first attempt's cause`);
+ }
+ assert.equal(
+ fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/model-output-diagnostics.js'), 'utf8'),
+ fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/model-output-diagnostics.js'), 'utf8'),
+ 'Chrome/Firefox output diagnostics must remain mirrored',
+ );
+});
+
+test('trace export: renders privacy-safe empty-response diagnostics', () => {
+ const runs = [{
+ run: { runId: 'empty-output', userMessage: 'Continue', model: 'test-model', status: 'empty_output' },
+ events: [{
+ runId: 'empty-output',
+ seq: 1,
+ kind: 'llm_response',
+ data: {
+ step: 7,
+ content: null,
+ toolCalls: [],
+ empty: true,
+ emptyReason: 'output_limit',
+ finishReason: 'length',
+ contentChars: 0,
+ toolCallCount: 0,
+ reasoningPresent: true,
+ reasoningChars: 0,
+ reasoningTokens: 4096,
+ outputTokens: 4096,
+ requestedMaxTokens: 4096,
+ recoveryAttempt: 2,
+ },
+ }],
+ }];
+ for (const [label, serialize] of [['chrome', tracesToMarkdown], ['firefox', tracesToMarkdownFx]]) {
+ const { markdown } = serialize(runs);
+ assert.match(markdown, /Empty model response: reason=output_limit 路 finish=length 路 output=4096 tokens 路 reasoning=4096 tokens 路 limit=4096 tokens 路 attempt=2 路 0 visible chars 路 0 tool calls/, `${label}: empty-response metadata remains hidden`);
+ const legacy = serialize([{
+ run: { runId: 'legacy-empty', userMessage: 'Continue', status: 'empty_output' },
+ events: [{ runId: 'legacy-empty', seq: 1, kind: 'llm_response', data: { content: null, toolCalls: [] } }],
+ }]).markdown;
+ assert.match(legacy, /Empty model response: reason=unknown/, `${label}: old empty events should stay visible without an invented cause`);
+ assert.doesNotMatch(legacy, /reason=provider_empty/, `${label}: legacy events cannot prove a provider-empty cause`);
+ }
+});
+
test('agent trace error classification: _traceErrorCodeFor maps failures to stable codes', () => {
for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) {
const agent = new AgentClass({});
@@ -9859,6 +9996,7 @@ test('trace recorder: turn/step boundary helpers and structured error codes are
const endRunBody = recorderSource.slice(recorderSource.indexOf('export async function endRun'), recorderSource.indexOf('/**\n * Repair trace records'));
assert.doesNotMatch(endRunBody, /_runWriteQueues\.delete\(runId\)/, `${browser}: finalization must let the queue owner release itself after later queued migrations settle`);
assert.match(recorderSource, /export function recordLLMRetry\([\s\S]*?_retryCount\(db, runId, step\)[\s\S]*?normalizeErrorCode\(code\)/, `${browser}: retry attempts are not derived from the durable event log`);
+ assert.match(recorderSource, /reasoningPresent[\s\S]*?reasoningTokens[\s\S]*?requestedMaxTokens/, `${browser}: privacy-safe model output diagnostics are not persisted`);
}
});
@@ -9877,6 +10015,10 @@ test('agent trace instrumentation: turn/step boundaries, retries-before-wait, an
assert.match(startTraceBody, /recordTurnStart\(runId, 0, \{ mode \}\)/, `${browser}: every started trace must receive turn_start, including planner exits`);
assert.match(nonStreamingBody, /recordStepStart\(runId, steps, \{\}\)/, `${browser}: non-streaming step_start missing`);
assert.match(streamingBody, /recordStepStart\(runId, steps, \{\}\)/, `${browser}: streaming step_start missing`);
+ assert.match(agentSource, /modelOutputDiagnostics\(result,[\s\S]*?requestedMaxTokens: chatOpts\.maxTokens/, `${browser}: non-streaming responses omit output diagnostics`);
+ assert.match(streamingBody, /recordLLMResponse\(runId, steps,[\s\S]*?\.\.\.outputDiagnostics/, `${browser}: streaming responses omit output diagnostics`);
+ assert.match(streamingBody, /finishReason = String\(/, `${browser}: streaming terminal reason is discarded`);
+ assert.doesNotMatch(agentSource, /raise the step limit in settings/, `${browser}: empty response guidance still blames the unrelated agent step limit`);
assert.match(nonStreamingBody, /_traceStepEndForResult\(result\)/, `${browser}: non-streaming output is not validated before step_end`);
assert.match(streamingBody, /const closeTraceStep = \(payload\) => \{[\s\S]*?traceStepClosed = true/, `${browser}: streaming step_end is not single-shot`);
const streamedClose = streamingBody.indexOf('closeTraceStep(this._traceStepEndForResult');
@@ -60558,7 +60700,7 @@ test('GPT-5.6 uses /responses while older and compatible-provider calls keep /ch
}), { status: 200, headers: { 'Content-Type': 'application/json' } });
}
return new Response(JSON.stringify({
- choices: [{ message: { content: 'legacy' } }],
+ choices: [{ message: { content: 'legacy' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
}), { status: 200, headers: { 'Content-Type': 'application/json' } });
};
@@ -60579,6 +60721,7 @@ test('GPT-5.6 uses /responses while older and compatible-provider calls keep /ch
response_items: [{ type: 'reasoning', encrypted_content: 'internal-only' }],
}], { tools: [tool] });
assert.equal(legacyResult.content, 'legacy');
+ assert.equal(legacyResult.finishReason, 'stop');
assert.equal(requests.at(-1).url, 'https://api.openai.com/v1/chat/completions');
assert.deepEqual(requests.at(-1).body.messages, [{ role: 'user', content: 'hello' }]);
assert.equal(requests.at(-1).body.tools[0].function.name, 'read_page');