From c2254863d54c6390ff46aacabee34c61ac12d73d Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Sun, 23 Aug 2026 19:41:08 +0300 Subject: [PATCH] fix: diagnose empty model responses --- src/chrome/src/agent/agent.js | 67 +++++++- .../src/agent/model-output-diagnostics.js | 115 ++++++++++++++ src/chrome/src/agent/trace-export.js | 25 ++- src/chrome/src/providers/openai.js | 1 + src/chrome/src/trace/recorder.js | 33 +++- src/chrome/src/ui/traces.js | 18 ++- src/firefox/src/agent/agent.js | 66 +++++++- .../src/agent/model-output-diagnostics.js | 115 ++++++++++++++ src/firefox/src/agent/trace-export.js | 25 ++- src/firefox/src/providers/openai.js | 1 + src/firefox/src/trace/recorder.js | 33 +++- src/firefox/src/ui/traces.js | 18 ++- test/run.js | 145 +++++++++++++++++- 13 files changed, 649 insertions(+), 13 deletions(-) create mode 100644 src/chrome/src/agent/model-output-diagnostics.js create mode 100644 src/firefox/src/agent/model-output-diagnostics.js diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index d72e6b1c5..5b0bbc095 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -103,6 +103,7 @@ import { } from '../providers/provider-compatibility.js'; import { extractFirstJsonObject } from './json-extract.js'; import { repairAssistantDisplayText, sanitizeText as sanitizePlannerText } from './text-sanitize.js'; +import { emptyOutputFailureMessage, modelOutputDiagnostics } from './model-output-diagnostics.js'; import { buildCustomSkillsPrompt, buildSkillLoaderDefinition, buildSkillToolDefinitions, buildSkillToolRegistry, getEligibleCustomSkills, getEligibleSkillCatalog, normalizeCustomSkills } from './skills.js'; import { publicMediaUrlNeedsExplicitTarget } from './public-media-url.js'; import { USER_MEMORY_DEFAULT_MAX_PROMPT_CHARS, formatUserMemoryPrompt, normalizeUserMemoryMaxPromptChars, normalizeUserMemoryStore } from './user-memory.js'; @@ -27659,6 +27660,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d model: provider.model, messageCount: prunedMessages.length, toolsCount: (chatOpts.tools || []).length, + requestedMaxTokens: chatOpts.maxTokens, ...(runOptions?.localWikipediaRag ? { localWikipediaRag: runOptions.localWikipediaRag } : {}), ...Agent._traceMediaCounts(prunedMessages), }, { @@ -27695,6 +27697,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d usage: result.usage, latencyMs: _llmLatency, model: provider.model, + ...modelOutputDiagnostics(result, { + requestedMaxTokens: chatOpts.maxTokens, + recoveryAttempt: emptyOutputRecoveryAttempted ? 2 : 1, + }), }); if (shouldOrderInteractiveAskTrace) await queueAskStreamingTraceWrite(writeResponseTrace); else writeResponseTrace(); @@ -28007,7 +28013,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d break; } // Second empty in a row: give up with a transparent message. - finalResponse = '[Agent emitted no output and no tool call, even after a recovery nudge. This usually means the task exceeded the current model\'s capability or context budget. Try a stronger model, raise the step limit in settings, or break the task into smaller parts.]'; + finalResponse = emptyOutputFailureMessage(modelOutputDiagnostics(result, { + requestedMaxTokens: 4096, + recoveryAttempt: 2, + })); _traceStatus = 'empty_output'; traceFailureCode = 'EMPTY_RESPONSE'; messages.push({ role: 'assistant', content: finalResponse }); @@ -28580,6 +28589,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d let hasToolCalls = false; let responseItems = null; let reasoningContent = ''; + let streamUsage = null; + let finishReason = ''; const streamOpts = this._cloudGenerationOptions(provider, { tools: provider.supportsTools && tools.length > 0 ? tools : undefined, @@ -28600,6 +28611,22 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d await closeTraceStep({ ok: false, code: 'COST_LIMIT' }); return finish(beforeCost, 'cost_limit'); } + if (runId) { + await trace.recordLLMRequest(runId, steps, { + providerClass: provider.constructor.name, + model: provider.model, + messageCount: prunedMessages.length, + toolsCount: (streamOpts.tools || []).length, + requestedMaxTokens: streamOpts.maxTokens, + ...(runOptions?.localWikipediaRag ? { localWikipediaRag: runOptions.localWikipediaRag } : {}), + ...Agent._traceMediaCounts(prunedMessages), + }, { + messages: prunedMessages, + tools: streamOpts.tools || [], + runtimeMode: mode, + }); + } + const _llmStart = Date.now(); let costStopMessage = ''; for await (const chunk of provider.chatStream(prunedMessages, streamOpts)) { @@ -28610,6 +28637,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } else if (chunk.type === 'reasoning') { reasoningContent += String(chunk.content || ''); } else if (chunk.type === 'usage') { + streamUsage = chunk.usage || streamUsage; costStopMessage = (await this._recordCostUsage(provider, chunk.usage, costState)) || costStopMessage; } else if (chunk.type === 'tool_call') { streamEmittedOutput = true; @@ -28641,14 +28669,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (Array.isArray(chunk.responseItems) && chunk.responseItems.length) { responseItems = chunk.responseItems; } + finishReason = String( + chunk.finishReason + ?? chunk.finish_reason + ?? chunk.stopReason + ?? chunk.stop_reason + ?? '', + ); + if (chunk.usage && !streamUsage) { + streamUsage = chunk.usage; + costStopMessage = (await this._recordCostUsage(provider, chunk.usage, costState)) || costStopMessage; + } break; } } fullText = Agent._stripReasoningTags(fullText); + const streamedToolCalls = hasToolCalls ? Object.values(toolCallsAccumulator) : []; + const outputDiagnostics = modelOutputDiagnostics({ + content: fullText, + toolCalls: streamedToolCalls, + reasoningContent, + usage: streamUsage, + finishReason, + responseItems, + }, { + requestedMaxTokens: streamOpts.maxTokens, + recoveryAttempt: emptyOutputRecoveryAttempted ? 2 : 1, + }); + if (runId) { + await trace.recordLLMResponse(runId, steps, { + content: fullText, + toolCalls: streamedToolCalls, + usage: streamUsage, + latencyMs: Date.now() - _llmStart, + model: provider.model, + ...outputDiagnostics, + }); + } closeTraceStep(this._traceStepEndForResult({ content: fullText, - toolCalls: hasToolCalls ? Object.values(toolCallsAccumulator) : [], + toolCalls: streamedToolCalls, })); // Match the non-streaming standalone profile: reinterpret LFM's @@ -28793,7 +28854,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d this._persist(tabId); return finish(scheduledResume.message, 'scheduled_resume'); } - const failMsg = '[Agent emitted no output and no tool call, even after a recovery nudge. This usually means the task exceeded the current model\'s capability or context budget. Try a stronger model, raise the step limit in settings, or break the task into smaller parts.]'; + const failMsg = emptyOutputFailureMessage(outputDiagnostics); messages.push({ role: 'assistant', content: failMsg }); onUpdate('warning', { message: failMsg }); this._persist(tabId); diff --git a/src/chrome/src/agent/model-output-diagnostics.js b/src/chrome/src/agent/model-output-diagnostics.js new file mode 100644 index 000000000..47ac13d58 --- /dev/null +++ b/src/chrome/src/agent/model-output-diagnostics.js @@ -0,0 +1,115 @@ +const OUTPUT_LIMIT_RE = /(?:^|[_\s-])(?:length|max(?:imum)?(?:[_\s-]*(?:output|completion))?[_\s-]*tokens?|output[_\s-]*limit|token[_\s-]*limit)(?:$|[_\s-])/i; +const CONTENT_FILTER_RE = /content[_\s-]*filter|safety|blocked|refusal/i; + +function finiteNonNegative(...values) { + for (const value of values) { + if (value == null || value === '') continue; + const number = Number(value); + if (Number.isFinite(number) && number >= 0) return Math.floor(number); + } + return null; +} + +function finitePositive(value) { + const number = Number(value); + return Number.isFinite(number) && number > 0 ? Math.floor(number) : null; +} + +function finishReasonFor(result) { + const raw = result?.raw || {}; + return String( + result?.finishReason + ?? result?.finish_reason + ?? result?.stopReason + ?? result?.stop_reason + ?? raw?.choices?.[0]?.finish_reason + ?? raw?.finish_reason + ?? raw?.stopReason + ?? raw?.stop_reason + ?? '', + ).replace(/[\u0000-\u001f\u007f]/g, '').trim().slice(0, 80); +} + +function usageFor(result) { + return result?.usage && typeof result.usage === 'object' + ? result.usage + : (result?.raw?.usage && typeof result.raw.usage === 'object' ? result.raw.usage : {}); +} + +function responseHasReasoningItem(result) { + const items = Array.isArray(result?.responseItems) + ? result.responseItems + : (Array.isArray(result?.raw?.output) ? result.raw.output : []); + return items.some((item) => { + if (item?.type === 'reasoning') return true; + if (item?.type !== 'webbrain_provider_replay' || !Array.isArray(item.content)) return false; + return item.content.some(block => block?.type === 'thinking' || block?.type === 'redacted_thinking'); + }); +} + +export function modelOutputDiagnostics(result, { requestedMaxTokens = null, recoveryAttempt = 0 } = {}) { + const contentChars = typeof result?.content === 'string' ? result.content.trim().length : 0; + const toolCallCount = Array.isArray(result?.toolCalls) ? result.toolCalls.length : 0; + const reasoningChars = typeof result?.reasoningContent === 'string' ? result.reasoningContent.length : 0; + const usage = usageFor(result); + const reasoningTokens = finiteNonNegative( + usage?.completion_tokens_details?.reasoning_tokens, + usage?.output_tokens_details?.reasoning_tokens, + usage?.reasoning_tokens, + ); + const outputTokens = finiteNonNegative( + usage?.completion_tokens, + usage?.output_tokens, + usage?.completionTokens, + usage?.outputTokens, + ); + const normalizedMaxTokens = finitePositive(requestedMaxTokens); + const finishReason = finishReasonFor(result); + const reasoningPresent = reasoningChars > 0 + || (reasoningTokens != null && reasoningTokens > 0) + || responseHasReasoningItem(result); + const empty = contentChars === 0 && toolCallCount === 0; + let emptyReason = null; + + if (empty) { + if (CONTENT_FILTER_RE.test(finishReason)) { + emptyReason = 'content_filter'; + } else if ( + OUTPUT_LIMIT_RE.test(` ${finishReason} `) + || (normalizedMaxTokens != null && outputTokens != null && outputTokens >= normalizedMaxTokens) + ) { + emptyReason = 'output_limit'; + } else if (reasoningPresent) { + emptyReason = 'reasoning_only'; + } else { + emptyReason = 'provider_empty'; + } + } + + return { + empty, + emptyReason, + finishReason: finishReason || null, + contentChars, + toolCallCount, + reasoningPresent, + reasoningChars, + reasoningTokens, + outputTokens, + requestedMaxTokens: normalizedMaxTokens, + recoveryAttempt: Number.isInteger(recoveryAttempt) && recoveryAttempt > 0 ? recoveryAttempt : 0, + }; +} + +export function emptyOutputFailureMessage(diagnostics = {}) { + switch (diagnostics.emptyReason) { + case 'output_limit': + return '[The model reached its configured response-token limit without producing visible text or a tool call, even after a recovery nudge. Reduce reasoning effort or choose another model/provider.]'; + case 'reasoning_only': + return '[The latest model response contained reasoning but no visible answer or tool call after the earlier empty response. Reduce reasoning effort or choose another model/provider.]'; + case 'content_filter': + return '[The provider filtered the latest response after an earlier empty response, so no visible text or tool call was returned. Revise the request or try another provider.]'; + default: + return '[The provider returned an empty completion with no visible text or tool call twice. Retry later or choose another model/provider.]'; + } +} diff --git a/src/chrome/src/agent/trace-export.js b/src/chrome/src/agent/trace-export.js index 452b85a94..c62b56abc 100644 --- a/src/chrome/src/agent/trace-export.js +++ b/src/chrome/src/agent/trace-export.js @@ -97,6 +97,24 @@ function renderLosslessRequest(messages, tools) { function oneLine(t) { return String(t ?? '').replace(/\s+/g, ' ').trim(); } function humanSize(n) { return n >= 1024 ? `${(n / 1024).toFixed(1)}kb` : `${n}b`; } +function renderEmptyModelResponse(data) { + const details = [ + `reason=${oneLine(data.emptyReason || 'unknown')}`, + data.finishReason ? `finish=${oneLine(data.finishReason)}` : '', + Number.isInteger(data.outputTokens) ? `output=${data.outputTokens} tokens` : '', + Number.isInteger(data.reasoningTokens) + ? `reasoning=${data.reasoningTokens} tokens` + : (Number.isInteger(data.reasoningChars) && data.reasoningChars > 0 + ? `reasoning=${data.reasoningChars} chars` + : (data.reasoningPresent === true ? 'reasoning=present' : '')), + Number.isInteger(data.requestedMaxTokens) ? `limit=${data.requestedMaxTokens} tokens` : '', + Number.isInteger(data.recoveryAttempt) ? `attempt=${data.recoveryAttempt}` : '', + `${Number.isInteger(data.contentChars) ? data.contentChars : 0} visible chars`, + `${Number.isInteger(data.toolCallCount) ? data.toolCallCount : 0} tool calls`, + ].filter(Boolean).join(' 路 '); + return `- 馃 Empty model response: ${details}\n`; +} + function truncate(text, limit) { const s = String(text ?? ''); if (s.length <= limit) return s; @@ -274,7 +292,12 @@ export function tracesToMarkdown(runsWithEvents, { md += `- 馃 Model request: ${Number(d.messageCount) || 0} messages 路 ${Number(d.toolsCount) || 0} tools${media ? ` 路 ${media}` : ''}${renderLocalWikipediaRag(d.localWikipediaRag)}${renderPromptProvenance(d.promptProvenance)}${d.lossless === true ? renderLosslessRequest(d.messages, d.tools) : ''}\n`; } else if (ev.kind === 'llm_response') { const content = String(d.content || '').trim(); - if (!content) continue; + if (!content) { + if (!Array.isArray(d.toolCalls) || d.toolCalls.length === 0) { + md += renderEmptyModelResponse(d); + } + continue; + } // Plan-before-Act runs record the planner call with phase:'planner'; keep // it (derails often start in the plan) but label it and preserve its shape. if (d.phase === 'planner') { diff --git a/src/chrome/src/providers/openai.js b/src/chrome/src/providers/openai.js index a6bac3426..23c5c3968 100644 --- a/src/chrome/src/providers/openai.js +++ b/src/chrome/src/providers/openai.js @@ -978,6 +978,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider { reasoningContent: message?.reasoning_content || message?.reasoning || '', toolCalls: message?.tool_calls || null, usage: data.usage || null, + finishReason: String(data?.choices?.[0]?.finish_reason || ''), raw: data, }; } diff --git a/src/chrome/src/trace/recorder.js b/src/chrome/src/trace/recorder.js index 50ef4ac86..36503ae81 100644 --- a/src/chrome/src/trace/recorder.js +++ b/src/chrome/src/trace/recorder.js @@ -665,7 +665,27 @@ export function recordLLMRequest(runId, step, payload, provenanceInput = null) { }); } -export function recordLLMResponse(runId, step, { content, toolCalls, usage, latencyMs, model, phase, attempt, repair }) { +export function recordLLMResponse(runId, step, { + content, + toolCalls, + usage, + latencyMs, + model, + phase, + attempt, + repair, + empty, + emptyReason, + finishReason, + contentChars, + toolCallCount, + reasoningPresent, + reasoningChars, + reasoningTokens, + outputTokens, + requestedMaxTokens, + recoveryAttempt, +}) { return _appendEvent(runId, 'llm_response', { step, content: content || null, @@ -682,6 +702,17 @@ export function recordLLMResponse(runId, step, { content, toolCalls, usage, late ...(phase ? { phase } : {}), ...(Number.isInteger(attempt) ? { attempt } : {}), ...(repair === true ? { repair: true } : {}), + ...(typeof empty === 'boolean' ? { empty } : {}), + ...(emptyReason ? { emptyReason } : {}), + ...(finishReason ? { finishReason } : {}), + ...(Number.isInteger(contentChars) ? { contentChars } : {}), + ...(Number.isInteger(toolCallCount) ? { toolCallCount } : {}), + ...(typeof reasoningPresent === 'boolean' ? { reasoningPresent } : {}), + ...(Number.isInteger(reasoningChars) ? { reasoningChars } : {}), + ...(Number.isInteger(reasoningTokens) ? { reasoningTokens } : {}), + ...(Number.isInteger(outputTokens) ? { outputTokens } : {}), + ...(Number.isInteger(requestedMaxTokens) ? { requestedMaxTokens } : {}), + ...(Number.isInteger(recoveryAttempt) && recoveryAttempt > 0 ? { recoveryAttempt } : {}), }); } diff --git a/src/chrome/src/ui/traces.js b/src/chrome/src/ui/traces.js index 59403934d..48f73f2ad 100644 --- a/src/chrome/src/ui/traces.js +++ b/src/chrome/src/ui/traces.js @@ -525,9 +525,10 @@ function renderEvent(ev, shotCache, compact, objectUrls = new Set()) { const usage = u ? `${(u.prompt_tokens || 0).toLocaleString()} in / ${(u.completion_tokens || 0).toLocaleString()} out` : ''; const lat = ev.data?.latencyMs != null ? `${ev.data.latencyMs} ms` : ''; const content = ev.data?.content; + const hasVisibleContent = typeof content === 'string' ? content.trim().length > 0 : !!content; const toolCalls = ev.data?.toolCalls || []; let body = ''; - if (content) { + if (hasVisibleContent) { body += `
${escapeHtml(content)}
`; } if (toolCalls.length > 0) { @@ -538,6 +539,21 @@ function renderEvent(ev, shotCache, compact, objectUrls = new Set()) { }).join(''); body += `
${tcList}
`; } + if (!hasVisibleContent && toolCalls.length === 0) { + const emptyDetails = [ + `reason=${ev.data?.emptyReason || 'unknown'}`, + ev.data?.finishReason ? `finish=${ev.data.finishReason}` : '', + Number.isInteger(ev.data?.outputTokens) ? `output_tokens=${ev.data.outputTokens}` : '', + Number.isInteger(ev.data?.reasoningTokens) + ? `reasoning_tokens=${ev.data.reasoningTokens}` + : (ev.data?.reasoningPresent === true ? 'reasoning_present=true' : ''), + Number.isInteger(ev.data?.requestedMaxTokens) ? `requested_max_tokens=${ev.data.requestedMaxTokens}` : '', + Number.isInteger(ev.data?.recoveryAttempt) ? `attempt=${ev.data.recoveryAttempt}` : '', + `content_chars=${Number.isInteger(ev.data?.contentChars) ? ev.data.contentChars : 0}`, + `tool_calls=${Number.isInteger(ev.data?.toolCallCount) ? ev.data.toolCallCount : 0}`, + ].filter(Boolean).join(' 路 '); + body = `
EMPTY_RESPONSE 路 ${escapeHtml(emptyDetails)}
`; + } return `
${escapeHtml(t('tr.event.llm_response'))}${stepBadge}${usage}${lat}${ts}
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 4c32ba97b..0dd252e54 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -95,6 +95,7 @@ import { } from '../providers/provider-compatibility.js'; import { extractFirstJsonObject } from './json-extract.js'; import { repairAssistantDisplayText, sanitizeText as sanitizePlannerText } from './text-sanitize.js'; +import { emptyOutputFailureMessage, modelOutputDiagnostics } from './model-output-diagnostics.js'; import { buildCustomSkillsPrompt, buildSkillLoaderDefinition, buildSkillToolDefinitions, buildSkillToolRegistry, getEligibleCustomSkills, getEligibleSkillCatalog, normalizeCustomSkills } from './skills.js'; import { publicMediaUrlNeedsExplicitTarget } from './public-media-url.js'; import { USER_MEMORY_DEFAULT_MAX_PROMPT_CHARS, formatUserMemoryPrompt, normalizeUserMemoryMaxPromptChars, normalizeUserMemoryStore } from './user-memory.js'; @@ -21274,6 +21275,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d model: provider.model, messageCount: prunedMessages.length, toolsCount: (chatOpts.tools || []).length, + requestedMaxTokens: chatOpts.maxTokens, ...Agent._traceMediaCounts(prunedMessages), }, { messages: prunedMessages, @@ -21299,6 +21301,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d usage: result.usage, latencyMs: Date.now() - _llmStart, model: provider.model, + ...modelOutputDiagnostics(result, { + requestedMaxTokens: chatOpts.maxTokens, + recoveryAttempt: emptyOutputRecoveryAttempted ? 2 : 1, + }), }); try { if (shouldOrderInteractiveAskTrace) await queueAskStreamingTraceWrite(writeResponseTrace); @@ -21546,7 +21552,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d this._persist(tabId); break; } - finalResponse = '[Agent emitted no output and no tool call, even after a recovery nudge. This usually means the task exceeded the current model\'s capability or context budget. Try a stronger model, raise the step limit in settings, or break the task into smaller parts.]'; + finalResponse = emptyOutputFailureMessage(modelOutputDiagnostics(result, { + requestedMaxTokens: 4096, + recoveryAttempt: 2, + })); _traceStatus = 'empty_output'; traceFailureCode = 'EMPTY_RESPONSE'; messages.push({ role: 'assistant', content: finalResponse }); @@ -22033,6 +22042,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d let hasToolCalls = false; let responseItems = null; let reasoningContent = ''; + let streamUsage = null; + let finishReason = ''; const streamOpts = this._cloudGenerationOptions(provider, { tools: provider.supportsTools && tools.length > 0 ? tools : undefined, @@ -22053,6 +22064,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d await closeTraceStep({ ok: false, code: 'COST_LIMIT' }); return finish(beforeCost, 'cost_limit'); } + if (runId) { + await trace.recordLLMRequest(runId, steps, { + providerClass: provider.constructor.name, + model: provider.model, + messageCount: prunedMessages.length, + toolsCount: (streamOpts.tools || []).length, + requestedMaxTokens: streamOpts.maxTokens, + ...Agent._traceMediaCounts(prunedMessages), + }, { + messages: prunedMessages, + tools: streamOpts.tools || [], + runtimeMode: mode, + }); + } + const _llmStart = Date.now(); let costStopMessage = ''; for await (const chunk of provider.chatStream(prunedMessages, streamOpts)) { @@ -22063,6 +22089,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } else if (chunk.type === 'reasoning') { reasoningContent += String(chunk.content || ''); } else if (chunk.type === 'usage') { + streamUsage = chunk.usage || streamUsage; costStopMessage = (await this._recordCostUsage(provider, chunk.usage, costState)) || costStopMessage; } else if (chunk.type === 'tool_call') { streamEmittedOutput = true; @@ -22094,14 +22121,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d if (Array.isArray(chunk.responseItems) && chunk.responseItems.length) { responseItems = chunk.responseItems; } + finishReason = String( + chunk.finishReason + ?? chunk.finish_reason + ?? chunk.stopReason + ?? chunk.stop_reason + ?? '', + ); + if (chunk.usage && !streamUsage) { + streamUsage = chunk.usage; + costStopMessage = (await this._recordCostUsage(provider, chunk.usage, costState)) || costStopMessage; + } break; } } fullText = Agent._stripReasoningTags(fullText); + const streamedToolCalls = hasToolCalls ? Object.values(toolCallsAccumulator) : []; + const outputDiagnostics = modelOutputDiagnostics({ + content: fullText, + toolCalls: streamedToolCalls, + reasoningContent, + usage: streamUsage, + finishReason, + responseItems, + }, { + requestedMaxTokens: streamOpts.maxTokens, + recoveryAttempt: emptyOutputRecoveryAttempted ? 2 : 1, + }); + if (runId) { + await trace.recordLLMResponse(runId, steps, { + content: fullText, + toolCalls: streamedToolCalls, + usage: streamUsage, + latencyMs: Date.now() - _llmStart, + model: provider.model, + ...outputDiagnostics, + }); + } closeTraceStep(this._traceStepEndForResult({ content: fullText, - toolCalls: hasToolCalls ? Object.values(toolCallsAccumulator) : [], + toolCalls: streamedToolCalls, })); // Fallback: parse tool calls from streamed text if structured calls are missing. @@ -22198,7 +22258,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d this._persist(tabId); return finish(scheduledResume.message, 'scheduled_resume'); } - const failMsg = '[Agent emitted no output and no tool call, even after a recovery nudge. This usually means the task exceeded the current model\'s capability or context budget. Try a stronger model, raise the step limit in settings, or break the task into smaller parts.]'; + const failMsg = emptyOutputFailureMessage(outputDiagnostics); messages.push({ role: 'assistant', content: failMsg }); onUpdate('warning', { message: failMsg }); this._persist(tabId); diff --git a/src/firefox/src/agent/model-output-diagnostics.js b/src/firefox/src/agent/model-output-diagnostics.js new file mode 100644 index 000000000..47ac13d58 --- /dev/null +++ b/src/firefox/src/agent/model-output-diagnostics.js @@ -0,0 +1,115 @@ +const OUTPUT_LIMIT_RE = /(?:^|[_\s-])(?:length|max(?:imum)?(?:[_\s-]*(?:output|completion))?[_\s-]*tokens?|output[_\s-]*limit|token[_\s-]*limit)(?:$|[_\s-])/i; +const CONTENT_FILTER_RE = /content[_\s-]*filter|safety|blocked|refusal/i; + +function finiteNonNegative(...values) { + for (const value of values) { + if (value == null || value === '') continue; + const number = Number(value); + if (Number.isFinite(number) && number >= 0) return Math.floor(number); + } + return null; +} + +function finitePositive(value) { + const number = Number(value); + return Number.isFinite(number) && number > 0 ? Math.floor(number) : null; +} + +function finishReasonFor(result) { + const raw = result?.raw || {}; + return String( + result?.finishReason + ?? result?.finish_reason + ?? result?.stopReason + ?? result?.stop_reason + ?? raw?.choices?.[0]?.finish_reason + ?? raw?.finish_reason + ?? raw?.stopReason + ?? raw?.stop_reason + ?? '', + ).replace(/[\u0000-\u001f\u007f]/g, '').trim().slice(0, 80); +} + +function usageFor(result) { + return result?.usage && typeof result.usage === 'object' + ? result.usage + : (result?.raw?.usage && typeof result.raw.usage === 'object' ? result.raw.usage : {}); +} + +function responseHasReasoningItem(result) { + const items = Array.isArray(result?.responseItems) + ? result.responseItems + : (Array.isArray(result?.raw?.output) ? result.raw.output : []); + return items.some((item) => { + if (item?.type === 'reasoning') return true; + if (item?.type !== 'webbrain_provider_replay' || !Array.isArray(item.content)) return false; + return item.content.some(block => block?.type === 'thinking' || block?.type === 'redacted_thinking'); + }); +} + +export function modelOutputDiagnostics(result, { requestedMaxTokens = null, recoveryAttempt = 0 } = {}) { + const contentChars = typeof result?.content === 'string' ? result.content.trim().length : 0; + const toolCallCount = Array.isArray(result?.toolCalls) ? result.toolCalls.length : 0; + const reasoningChars = typeof result?.reasoningContent === 'string' ? result.reasoningContent.length : 0; + const usage = usageFor(result); + const reasoningTokens = finiteNonNegative( + usage?.completion_tokens_details?.reasoning_tokens, + usage?.output_tokens_details?.reasoning_tokens, + usage?.reasoning_tokens, + ); + const outputTokens = finiteNonNegative( + usage?.completion_tokens, + usage?.output_tokens, + usage?.completionTokens, + usage?.outputTokens, + ); + const normalizedMaxTokens = finitePositive(requestedMaxTokens); + const finishReason = finishReasonFor(result); + const reasoningPresent = reasoningChars > 0 + || (reasoningTokens != null && reasoningTokens > 0) + || responseHasReasoningItem(result); + const empty = contentChars === 0 && toolCallCount === 0; + let emptyReason = null; + + if (empty) { + if (CONTENT_FILTER_RE.test(finishReason)) { + emptyReason = 'content_filter'; + } else if ( + OUTPUT_LIMIT_RE.test(` ${finishReason} `) + || (normalizedMaxTokens != null && outputTokens != null && outputTokens >= normalizedMaxTokens) + ) { + emptyReason = 'output_limit'; + } else if (reasoningPresent) { + emptyReason = 'reasoning_only'; + } else { + emptyReason = 'provider_empty'; + } + } + + return { + empty, + emptyReason, + finishReason: finishReason || null, + contentChars, + toolCallCount, + reasoningPresent, + reasoningChars, + reasoningTokens, + outputTokens, + requestedMaxTokens: normalizedMaxTokens, + recoveryAttempt: Number.isInteger(recoveryAttempt) && recoveryAttempt > 0 ? recoveryAttempt : 0, + }; +} + +export function emptyOutputFailureMessage(diagnostics = {}) { + switch (diagnostics.emptyReason) { + case 'output_limit': + return '[The model reached its configured response-token limit without producing visible text or a tool call, even after a recovery nudge. Reduce reasoning effort or choose another model/provider.]'; + case 'reasoning_only': + return '[The latest model response contained reasoning but no visible answer or tool call after the earlier empty response. Reduce reasoning effort or choose another model/provider.]'; + case 'content_filter': + return '[The provider filtered the latest response after an earlier empty response, so no visible text or tool call was returned. Revise the request or try another provider.]'; + default: + return '[The provider returned an empty completion with no visible text or tool call twice. Retry later or choose another model/provider.]'; + } +} diff --git a/src/firefox/src/agent/trace-export.js b/src/firefox/src/agent/trace-export.js index c738cbd50..a5b6c8a62 100644 --- a/src/firefox/src/agent/trace-export.js +++ b/src/firefox/src/agent/trace-export.js @@ -103,6 +103,24 @@ function renderLosslessRequest(messages, tools) { function oneLine(t) { return String(t ?? '').replace(/\s+/g, ' ').trim(); } function humanSize(n) { return n >= 1024 ? `${(n / 1024).toFixed(1)}kb` : `${n}b`; } +function renderEmptyModelResponse(data) { + const details = [ + `reason=${oneLine(data.emptyReason || 'unknown')}`, + data.finishReason ? `finish=${oneLine(data.finishReason)}` : '', + Number.isInteger(data.outputTokens) ? `output=${data.outputTokens} tokens` : '', + Number.isInteger(data.reasoningTokens) + ? `reasoning=${data.reasoningTokens} tokens` + : (Number.isInteger(data.reasoningChars) && data.reasoningChars > 0 + ? `reasoning=${data.reasoningChars} chars` + : (data.reasoningPresent === true ? 'reasoning=present' : '')), + Number.isInteger(data.requestedMaxTokens) ? `limit=${data.requestedMaxTokens} tokens` : '', + Number.isInteger(data.recoveryAttempt) ? `attempt=${data.recoveryAttempt}` : '', + `${Number.isInteger(data.contentChars) ? data.contentChars : 0} visible chars`, + `${Number.isInteger(data.toolCallCount) ? data.toolCallCount : 0} tool calls`, + ].filter(Boolean).join(' 路 '); + return `- 馃 Empty model response: ${details}\n`; +} + function truncate(text, limit) { const s = String(text ?? ''); if (s.length <= limit) return s; @@ -280,7 +298,12 @@ export function tracesToMarkdown(runsWithEvents, { md += `- 馃 Model request: ${Number(d.messageCount) || 0} messages 路 ${Number(d.toolsCount) || 0} tools${media ? ` 路 ${media}` : ''}${renderLocalWikipediaRag(d.localWikipediaRag)}${renderPromptProvenance(d.promptProvenance)}${d.lossless === true ? renderLosslessRequest(d.messages, d.tools) : ''}\n`; } else if (ev.kind === 'llm_response') { const content = String(d.content || '').trim(); - if (!content) continue; + if (!content) { + if (!Array.isArray(d.toolCalls) || d.toolCalls.length === 0) { + md += renderEmptyModelResponse(d); + } + continue; + } // Plan-before-Act runs record the planner call with phase:'planner'; keep // it (derails often start in the plan) but label it and preserve its shape. if (d.phase === 'planner') { diff --git a/src/firefox/src/providers/openai.js b/src/firefox/src/providers/openai.js index e47d77ccc..b23af0f03 100644 --- a/src/firefox/src/providers/openai.js +++ b/src/firefox/src/providers/openai.js @@ -978,6 +978,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider { reasoningContent: message?.reasoning_content || message?.reasoning || '', toolCalls: message?.tool_calls || null, usage: data.usage || null, + finishReason: String(data?.choices?.[0]?.finish_reason || ''), raw: data, }; } diff --git a/src/firefox/src/trace/recorder.js b/src/firefox/src/trace/recorder.js index 272057f52..e5ac0a51a 100644 --- a/src/firefox/src/trace/recorder.js +++ b/src/firefox/src/trace/recorder.js @@ -648,7 +648,27 @@ export function recordLLMRequest(runId, step, payload, provenanceInput = null) { }); } -export function recordLLMResponse(runId, step, { content, toolCalls, usage, latencyMs, model, phase, attempt, repair }) { +export function recordLLMResponse(runId, step, { + content, + toolCalls, + usage, + latencyMs, + model, + phase, + attempt, + repair, + empty, + emptyReason, + finishReason, + contentChars, + toolCallCount, + reasoningPresent, + reasoningChars, + reasoningTokens, + outputTokens, + requestedMaxTokens, + recoveryAttempt, +}) { return _appendEvent(runId, 'llm_response', { step, content: content || null, @@ -665,6 +685,17 @@ export function recordLLMResponse(runId, step, { content, toolCalls, usage, late ...(phase ? { phase } : {}), ...(Number.isInteger(attempt) ? { attempt } : {}), ...(repair === true ? { repair: true } : {}), + ...(typeof empty === 'boolean' ? { empty } : {}), + ...(emptyReason ? { emptyReason } : {}), + ...(finishReason ? { finishReason } : {}), + ...(Number.isInteger(contentChars) ? { contentChars } : {}), + ...(Number.isInteger(toolCallCount) ? { toolCallCount } : {}), + ...(typeof reasoningPresent === 'boolean' ? { reasoningPresent } : {}), + ...(Number.isInteger(reasoningChars) ? { reasoningChars } : {}), + ...(Number.isInteger(reasoningTokens) ? { reasoningTokens } : {}), + ...(Number.isInteger(outputTokens) ? { outputTokens } : {}), + ...(Number.isInteger(requestedMaxTokens) ? { requestedMaxTokens } : {}), + ...(Number.isInteger(recoveryAttempt) && recoveryAttempt > 0 ? { recoveryAttempt } : {}), }); } diff --git a/src/firefox/src/ui/traces.js b/src/firefox/src/ui/traces.js index f2aaa4d1c..d98c42ea0 100644 --- a/src/firefox/src/ui/traces.js +++ b/src/firefox/src/ui/traces.js @@ -525,9 +525,10 @@ function renderEvent(ev, shotCache, compact, objectUrls = new Set()) { const usage = u ? `${(u.prompt_tokens || 0).toLocaleString()} in / ${(u.completion_tokens || 0).toLocaleString()} out` : ''; const lat = ev.data?.latencyMs != null ? `${ev.data.latencyMs} ms` : ''; const content = ev.data?.content; + const hasVisibleContent = typeof content === 'string' ? content.trim().length > 0 : !!content; const toolCalls = ev.data?.toolCalls || []; let body = ''; - if (content) { + if (hasVisibleContent) { body += `
${escapeHtml(content)}
`; } if (toolCalls.length > 0) { @@ -538,6 +539,21 @@ function renderEvent(ev, shotCache, compact, objectUrls = new Set()) { }).join(''); body += `
${tcList}
`; } + if (!hasVisibleContent && toolCalls.length === 0) { + const emptyDetails = [ + `reason=${ev.data?.emptyReason || 'unknown'}`, + ev.data?.finishReason ? `finish=${ev.data.finishReason}` : '', + Number.isInteger(ev.data?.outputTokens) ? `output_tokens=${ev.data.outputTokens}` : '', + Number.isInteger(ev.data?.reasoningTokens) + ? `reasoning_tokens=${ev.data.reasoningTokens}` + : (ev.data?.reasoningPresent === true ? 'reasoning_present=true' : ''), + Number.isInteger(ev.data?.requestedMaxTokens) ? `requested_max_tokens=${ev.data.requestedMaxTokens}` : '', + Number.isInteger(ev.data?.recoveryAttempt) ? `attempt=${ev.data.recoveryAttempt}` : '', + `content_chars=${Number.isInteger(ev.data?.contentChars) ? ev.data.contentChars : 0}`, + `tool_calls=${Number.isInteger(ev.data?.toolCallCount) ? ev.data.toolCallCount : 0}`, + ].filter(Boolean).join(' 路 '); + body = `
EMPTY_RESPONSE 路 ${escapeHtml(emptyDetails)}
`; + } return `
${escapeHtml(t('tr.event.llm_response'))}${stepBadge}${usage}${lat}${ts}
diff --git a/test/run.js b/test/run.js index 191842037..6a5437032 100644 --- a/test/run.js +++ b/test/run.js @@ -292,6 +292,12 @@ const { tracesToMarkdown, sanitizeTraceExport } = await import( const { tracesToMarkdown: tracesToMarkdownFx, sanitizeTraceExport: sanitizeTraceExportFx } = await import( 'file://' + path.join(ROOT, 'src/firefox/src/agent/trace-export.js').replace(/\\/g, '/') ); +const ModelOutputDiagnosticsCh = await import( + 'file://' + path.join(ROOT, 'src/chrome/src/agent/model-output-diagnostics.js').replace(/\\/g, '/') +); +const ModelOutputDiagnosticsFx = await import( + 'file://' + path.join(ROOT, 'src/firefox/src/agent/model-output-diagnostics.js').replace(/\\/g, '/') +); const Utf8BudgetCh = await import( 'file://' + path.join(ROOT, 'src/chrome/src/trace/utf8-budget.js').replace(/\\/g, '/') ); @@ -9794,6 +9800,9 @@ test('trace UI: renders the trajectory rows before the detailed event timeline', assert.match(traces, /getSessionStats\(run\.conversationId\)/, `${browser}: conversation panel does not read indexed session stats`); assert.match(traces, /trajectory-table/, `${browser}: trajectory table class missing from renderer`); assert.match(traces, /renderStepTrajectory\(events, compact\)/, `${browser}: run view does not render the trajectory before details`); + assert.match(traces, /EMPTY_RESPONSE 路 \$\{escapeHtml\(emptyDetails\)\}/, `${browser}: empty model responses have no visible diagnostics`); + assert.match(traces, /const hasVisibleContent = typeof content === 'string' \? content\.trim\(\)\.length > 0 : !!content/, `${browser}: whitespace-only responses are not rendered as empty`); + assert.match(traces, /emptyReason \|\| 'unknown'/, `${browser}: legacy empty responses should not receive an invented cause`); assert.match(html, /\.trajectory-table|\.trajectory-row/, `${browser}: trajectory table styles missing`); assert.match(html, /\.conv-summary/, `${browser}: conversation statistics style missing`); } @@ -9823,6 +9832,134 @@ test('trace UI: renders collapsed session lineage groups with bounded-result war } }); +test('model output diagnostics: classifies empty completions without retaining reasoning text', () => { + for (const [label, diagnostics] of [ + ['chrome', ModelOutputDiagnosticsCh], + ['firefox', ModelOutputDiagnosticsFx], + ]) { + const limited = diagnostics.modelOutputDiagnostics({ + content: '', + toolCalls: [], + reasoningContent: 'private reasoning text', + usage: { + completion_tokens: 4096, + completion_tokens_details: { reasoning_tokens: 4096 }, + }, + finishReason: 'length', + }, { requestedMaxTokens: 4096, recoveryAttempt: 2 }); + assert.deepEqual(limited, { + empty: true, + emptyReason: 'output_limit', + finishReason: 'length', + contentChars: 0, + toolCallCount: 0, + reasoningPresent: true, + reasoningChars: 22, + reasoningTokens: 4096, + outputTokens: 4096, + requestedMaxTokens: 4096, + recoveryAttempt: 2, + }, `${label}: output-limit diagnostics changed`); + assert.doesNotMatch(JSON.stringify(limited), /private reasoning text/, `${label}: reasoning text leaked into diagnostics`); + assert.match(diagnostics.emptyOutputFailureMessage(limited), /response-token limit/i, `${label}: output-limit guidance missing`); + assert.doesNotMatch(diagnostics.emptyOutputFailureMessage(limited), /step limit/i, `${label}: model output failures must not blame the agent step limit`); + assert.doesNotMatch(diagnostics.emptyOutputFailureMessage(limited), /increase the response-token limit/i, `${label}: output-limit guidance must not recommend an unavailable setting`); + + assert.equal(diagnostics.modelOutputDiagnostics({ + content: '', + toolCalls: null, + reasoningContent: 'thought', + usage: { completion_tokens: 12 }, + }, { requestedMaxTokens: 4096 }).emptyReason, 'reasoning_only', `${label}: reasoning-only response not classified`); + assert.equal(diagnostics.modelOutputDiagnostics({ + content: '', + toolCalls: [], + responseItems: [{ type: 'reasoning', encrypted_content: 'opaque' }], + }).emptyReason, 'reasoning_only', `${label}: encrypted reasoning item not detected without exposing it`); + assert.equal(diagnostics.modelOutputDiagnostics({ + content: '', + toolCalls: [], + usage: { completion_tokens: 42 }, + responseItems: [{ + type: 'webbrain_provider_replay', + content: [{ type: 'redacted_thinking', data: 'opaque' }], + }], + }, { requestedMaxTokens: 4096 }).emptyReason, 'reasoning_only', `${label}: provider replay reasoning envelope not detected`); + assert.equal(diagnostics.modelOutputDiagnostics({ + content: '', + toolCalls: [], + finishReason: 'content_filter', + }).emptyReason, 'content_filter', `${label}: filtered response not classified`); + assert.equal(diagnostics.modelOutputDiagnostics({ + content: '', + toolCalls: [], + usage: { completion_tokens: 0 }, + }, { requestedMaxTokens: 4096 }).emptyReason, 'provider_empty', `${label}: unexplained empty response not classified`); + assert.equal(diagnostics.modelOutputDiagnostics({ + content: '', + toolCalls: [], + raw: { choices: [{ finish_reason: 'max_tokens' }] }, + }).emptyReason, 'output_limit', `${label}: raw provider finish reason not classified`); + assert.equal(diagnostics.modelOutputDiagnostics({ + content: 'done', + toolCalls: [], + reasoningContent: 'thought', + }).emptyReason, null, `${label}: visible output must not receive an empty classification`); + assert.equal(diagnostics.modelOutputDiagnostics({ + content: '', + toolCalls: [{ id: 'call_1' }], + }).empty, false, `${label}: tool output must not be empty`); + const reasoningMessage = diagnostics.emptyOutputFailureMessage({ emptyReason: 'reasoning_only' }); + assert.match(reasoningMessage, /latest model response/i, `${label}: reasoning-only guidance should identify the observed attempt`); + assert.doesNotMatch(reasoningMessage, /twice/i, `${label}: reasoning-only guidance must not invent the first attempt's cause`); + const filteredMessage = diagnostics.emptyOutputFailureMessage({ emptyReason: 'content_filter' }); + assert.match(filteredMessage, /latest response/i, `${label}: filter guidance should identify the observed attempt`); + assert.doesNotMatch(filteredMessage, /both attempts/i, `${label}: filter guidance must not invent the first attempt's cause`); + } + assert.equal( + fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/model-output-diagnostics.js'), 'utf8'), + fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/model-output-diagnostics.js'), 'utf8'), + 'Chrome/Firefox output diagnostics must remain mirrored', + ); +}); + +test('trace export: renders privacy-safe empty-response diagnostics', () => { + const runs = [{ + run: { runId: 'empty-output', userMessage: 'Continue', model: 'test-model', status: 'empty_output' }, + events: [{ + runId: 'empty-output', + seq: 1, + kind: 'llm_response', + data: { + step: 7, + content: null, + toolCalls: [], + empty: true, + emptyReason: 'output_limit', + finishReason: 'length', + contentChars: 0, + toolCallCount: 0, + reasoningPresent: true, + reasoningChars: 0, + reasoningTokens: 4096, + outputTokens: 4096, + requestedMaxTokens: 4096, + recoveryAttempt: 2, + }, + }], + }]; + for (const [label, serialize] of [['chrome', tracesToMarkdown], ['firefox', tracesToMarkdownFx]]) { + const { markdown } = serialize(runs); + assert.match(markdown, /Empty model response: reason=output_limit 路 finish=length 路 output=4096 tokens 路 reasoning=4096 tokens 路 limit=4096 tokens 路 attempt=2 路 0 visible chars 路 0 tool calls/, `${label}: empty-response metadata remains hidden`); + const legacy = serialize([{ + run: { runId: 'legacy-empty', userMessage: 'Continue', status: 'empty_output' }, + events: [{ runId: 'legacy-empty', seq: 1, kind: 'llm_response', data: { content: null, toolCalls: [] } }], + }]).markdown; + assert.match(legacy, /Empty model response: reason=unknown/, `${label}: old empty events should stay visible without an invented cause`); + assert.doesNotMatch(legacy, /reason=provider_empty/, `${label}: legacy events cannot prove a provider-empty cause`); + } +}); + test('agent trace error classification: _traceErrorCodeFor maps failures to stable codes', () => { for (const [label, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { const agent = new AgentClass({}); @@ -9859,6 +9996,7 @@ test('trace recorder: turn/step boundary helpers and structured error codes are const endRunBody = recorderSource.slice(recorderSource.indexOf('export async function endRun'), recorderSource.indexOf('/**\n * Repair trace records')); assert.doesNotMatch(endRunBody, /_runWriteQueues\.delete\(runId\)/, `${browser}: finalization must let the queue owner release itself after later queued migrations settle`); assert.match(recorderSource, /export function recordLLMRetry\([\s\S]*?_retryCount\(db, runId, step\)[\s\S]*?normalizeErrorCode\(code\)/, `${browser}: retry attempts are not derived from the durable event log`); + assert.match(recorderSource, /reasoningPresent[\s\S]*?reasoningTokens[\s\S]*?requestedMaxTokens/, `${browser}: privacy-safe model output diagnostics are not persisted`); } }); @@ -9877,6 +10015,10 @@ test('agent trace instrumentation: turn/step boundaries, retries-before-wait, an assert.match(startTraceBody, /recordTurnStart\(runId, 0, \{ mode \}\)/, `${browser}: every started trace must receive turn_start, including planner exits`); assert.match(nonStreamingBody, /recordStepStart\(runId, steps, \{\}\)/, `${browser}: non-streaming step_start missing`); assert.match(streamingBody, /recordStepStart\(runId, steps, \{\}\)/, `${browser}: streaming step_start missing`); + assert.match(agentSource, /modelOutputDiagnostics\(result,[\s\S]*?requestedMaxTokens: chatOpts\.maxTokens/, `${browser}: non-streaming responses omit output diagnostics`); + assert.match(streamingBody, /recordLLMResponse\(runId, steps,[\s\S]*?\.\.\.outputDiagnostics/, `${browser}: streaming responses omit output diagnostics`); + assert.match(streamingBody, /finishReason = String\(/, `${browser}: streaming terminal reason is discarded`); + assert.doesNotMatch(agentSource, /raise the step limit in settings/, `${browser}: empty response guidance still blames the unrelated agent step limit`); assert.match(nonStreamingBody, /_traceStepEndForResult\(result\)/, `${browser}: non-streaming output is not validated before step_end`); assert.match(streamingBody, /const closeTraceStep = \(payload\) => \{[\s\S]*?traceStepClosed = true/, `${browser}: streaming step_end is not single-shot`); const streamedClose = streamingBody.indexOf('closeTraceStep(this._traceStepEndForResult'); @@ -60558,7 +60700,7 @@ test('GPT-5.6 uses /responses while older and compatible-provider calls keep /ch }), { status: 200, headers: { 'Content-Type': 'application/json' } }); } return new Response(JSON.stringify({ - choices: [{ message: { content: 'legacy' } }], + choices: [{ message: { content: 'legacy' }, finish_reason: 'stop' }], usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 }, }), { status: 200, headers: { 'Content-Type': 'application/json' } }); }; @@ -60579,6 +60721,7 @@ test('GPT-5.6 uses /responses while older and compatible-provider calls keep /ch response_items: [{ type: 'reasoning', encrypted_content: 'internal-only' }], }], { tools: [tool] }); assert.equal(legacyResult.content, 'legacy'); + assert.equal(legacyResult.finishReason, 'stop'); assert.equal(requests.at(-1).url, 'https://api.openai.com/v1/chat/completions'); assert.deepEqual(requests.at(-1).body.messages, [{ role: 'user', content: 'hello' }]); assert.equal(requests.at(-1).body.tools[0].function.name, 'read_page');