Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 64 additions & 3 deletions src/chrome/src/agent/agent.js
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ import {
} from '../providers/provider-compatibility.js';
import { extractFirstJsonObject } from './json-extract.js';
import { repairAssistantDisplayText, sanitizeText as sanitizePlannerText } from './text-sanitize.js';
import { emptyOutputFailureMessage, modelOutputDiagnostics } from './model-output-diagnostics.js';
import { buildCustomSkillsPrompt, buildSkillLoaderDefinition, buildSkillToolDefinitions, buildSkillToolRegistry, getEligibleCustomSkills, getEligibleSkillCatalog, normalizeCustomSkills } from './skills.js';
import { publicMediaUrlNeedsExplicitTarget } from './public-media-url.js';
import { USER_MEMORY_DEFAULT_MAX_PROMPT_CHARS, formatUserMemoryPrompt, normalizeUserMemoryMaxPromptChars, normalizeUserMemoryStore } from './user-memory.js';
Expand Down Expand Up @@ -27659,6 +27660,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
model: provider.model,
messageCount: prunedMessages.length,
toolsCount: (chatOpts.tools || []).length,
requestedMaxTokens: chatOpts.maxTokens,
...(runOptions?.localWikipediaRag ? { localWikipediaRag: runOptions.localWikipediaRag } : {}),
...Agent._traceMediaCounts(prunedMessages),
}, {
Expand Down Expand Up @@ -27695,6 +27697,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
usage: result.usage,
latencyMs: _llmLatency,
model: provider.model,
...modelOutputDiagnostics(result, {
requestedMaxTokens: chatOpts.maxTokens,
recoveryAttempt: emptyOutputRecoveryAttempted ? 2 : 1,
}),
});
if (shouldOrderInteractiveAskTrace) await queueAskStreamingTraceWrite(writeResponseTrace);
else writeResponseTrace();
Expand Down Expand Up @@ -28007,7 +28013,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
break;
}
// Second empty in a row: give up with a transparent message.
finalResponse = '[Agent emitted no output and no tool call, even after a recovery nudge. This usually means the task exceeded the current model\'s capability or context budget. Try a stronger model, raise the step limit in settings, or break the task into smaller parts.]';
finalResponse = emptyOutputFailureMessage(modelOutputDiagnostics(result, {
requestedMaxTokens: 4096,
recoveryAttempt: 2,
}));
_traceStatus = 'empty_output';
traceFailureCode = 'EMPTY_RESPONSE';
messages.push({ role: 'assistant', content: finalResponse });
Expand Down Expand Up @@ -28580,6 +28589,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
let hasToolCalls = false;
let responseItems = null;
let reasoningContent = '';
let streamUsage = null;
let finishReason = '';

const streamOpts = this._cloudGenerationOptions(provider, {
tools: provider.supportsTools && tools.length > 0 ? tools : undefined,
Expand All @@ -28600,6 +28611,22 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
await closeTraceStep({ ok: false, code: 'COST_LIMIT' });
return finish(beforeCost, 'cost_limit');
}
if (runId) {
await trace.recordLLMRequest(runId, steps, {
providerClass: provider.constructor.name,
model: provider.model,
messageCount: prunedMessages.length,
toolsCount: (streamOpts.tools || []).length,
requestedMaxTokens: streamOpts.maxTokens,
...(runOptions?.localWikipediaRag ? { localWikipediaRag: runOptions.localWikipediaRag } : {}),
...Agent._traceMediaCounts(prunedMessages),
}, {
messages: prunedMessages,
tools: streamOpts.tools || [],
runtimeMode: mode,
});
}
const _llmStart = Date.now();
let costStopMessage = '';

for await (const chunk of provider.chatStream(prunedMessages, streamOpts)) {
Expand All @@ -28610,6 +28637,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
} else if (chunk.type === 'reasoning') {
reasoningContent += String(chunk.content || '');
} else if (chunk.type === 'usage') {
streamUsage = chunk.usage || streamUsage;
costStopMessage = (await this._recordCostUsage(provider, chunk.usage, costState)) || costStopMessage;
} else if (chunk.type === 'tool_call') {
streamEmittedOutput = true;
Expand Down Expand Up @@ -28641,14 +28669,47 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (Array.isArray(chunk.responseItems) && chunk.responseItems.length) {
responseItems = chunk.responseItems;
}
finishReason = String(
chunk.finishReason
?? chunk.finish_reason
?? chunk.stopReason
?? chunk.stop_reason
?? '',
);
if (chunk.usage && !streamUsage) {
streamUsage = chunk.usage;
costStopMessage = (await this._recordCostUsage(provider, chunk.usage, costState)) || costStopMessage;
}
break;
}
}

fullText = Agent._stripReasoningTags(fullText);
const streamedToolCalls = hasToolCalls ? Object.values(toolCallsAccumulator) : [];
const outputDiagnostics = modelOutputDiagnostics({
content: fullText,
toolCalls: streamedToolCalls,
reasoningContent,
usage: streamUsage,
finishReason,
responseItems,
}, {
requestedMaxTokens: streamOpts.maxTokens,
recoveryAttempt: emptyOutputRecoveryAttempted ? 2 : 1,
});
if (runId) {
await trace.recordLLMResponse(runId, steps, {
content: fullText,
toolCalls: streamedToolCalls,
usage: streamUsage,
latencyMs: Date.now() - _llmStart,
model: provider.model,
...outputDiagnostics,
});
}
closeTraceStep(this._traceStepEndForResult({
content: fullText,
toolCalls: hasToolCalls ? Object.values(toolCallsAccumulator) : [],
toolCalls: streamedToolCalls,
}));

// Match the non-streaming standalone profile: reinterpret LFM's
Expand Down Expand Up @@ -28793,7 +28854,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this._persist(tabId);
return finish(scheduledResume.message, 'scheduled_resume');
}
const failMsg = '[Agent emitted no output and no tool call, even after a recovery nudge. This usually means the task exceeded the current model\'s capability or context budget. Try a stronger model, raise the step limit in settings, or break the task into smaller parts.]';
const failMsg = emptyOutputFailureMessage(outputDiagnostics);
messages.push({ role: 'assistant', content: failMsg });
onUpdate('warning', { message: failMsg });
this._persist(tabId);
Expand Down
115 changes: 115 additions & 0 deletions src/chrome/src/agent/model-output-diagnostics.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
const OUTPUT_LIMIT_RE = /(?:^|[_\s-])(?:length|max(?:imum)?(?:[_\s-]*(?:output|completion))?[_\s-]*tokens?|output[_\s-]*limit|token[_\s-]*limit)(?:$|[_\s-])/i;
const CONTENT_FILTER_RE = /content[_\s-]*filter|safety|blocked|refusal/i;

function finiteNonNegative(...values) {
for (const value of values) {
if (value == null || value === '') continue;
const number = Number(value);
if (Number.isFinite(number) && number >= 0) return Math.floor(number);
}
return null;
}

function finitePositive(value) {
const number = Number(value);
return Number.isFinite(number) && number > 0 ? Math.floor(number) : null;
}

function finishReasonFor(result) {
const raw = result?.raw || {};
return String(
result?.finishReason
?? result?.finish_reason
?? result?.stopReason
?? result?.stop_reason
?? raw?.choices?.[0]?.finish_reason
?? raw?.finish_reason
?? raw?.stopReason
?? raw?.stop_reason
?? '',
).replace(/[\u0000-\u001f\u007f]/g, '').trim().slice(0, 80);
}

function usageFor(result) {
return result?.usage && typeof result.usage === 'object'
? result.usage
: (result?.raw?.usage && typeof result.raw.usage === 'object' ? result.raw.usage : {});
}

function responseHasReasoningItem(result) {
const items = Array.isArray(result?.responseItems)
? result.responseItems
: (Array.isArray(result?.raw?.output) ? result.raw.output : []);
return items.some((item) => {
if (item?.type === 'reasoning') return true;
if (item?.type !== 'webbrain_provider_replay' || !Array.isArray(item.content)) return false;
return item.content.some(block => block?.type === 'thinking' || block?.type === 'redacted_thinking');
});
}

export function modelOutputDiagnostics(result, { requestedMaxTokens = null, recoveryAttempt = 0 } = {}) {
const contentChars = typeof result?.content === 'string' ? result.content.trim().length : 0;
const toolCallCount = Array.isArray(result?.toolCalls) ? result.toolCalls.length : 0;
const reasoningChars = typeof result?.reasoningContent === 'string' ? result.reasoningContent.length : 0;
const usage = usageFor(result);
const reasoningTokens = finiteNonNegative(
usage?.completion_tokens_details?.reasoning_tokens,
usage?.output_tokens_details?.reasoning_tokens,
usage?.reasoning_tokens,
);
const outputTokens = finiteNonNegative(
usage?.completion_tokens,
usage?.output_tokens,
usage?.completionTokens,
usage?.outputTokens,
);
const normalizedMaxTokens = finitePositive(requestedMaxTokens);
const finishReason = finishReasonFor(result);
const reasoningPresent = reasoningChars > 0
|| (reasoningTokens != null && reasoningTokens > 0)
|| responseHasReasoningItem(result);
const empty = contentChars === 0 && toolCallCount === 0;
let emptyReason = null;

if (empty) {
if (CONTENT_FILTER_RE.test(finishReason)) {
emptyReason = 'content_filter';
} else if (
OUTPUT_LIMIT_RE.test(` ${finishReason} `)
|| (normalizedMaxTokens != null && outputTokens != null && outputTokens >= normalizedMaxTokens)
) {
emptyReason = 'output_limit';
} else if (reasoningPresent) {
emptyReason = 'reasoning_only';
} else {
emptyReason = 'provider_empty';
}
}

return {
empty,
emptyReason,
finishReason: finishReason || null,
contentChars,
toolCallCount,
reasoningPresent,
reasoningChars,
reasoningTokens,
outputTokens,
requestedMaxTokens: normalizedMaxTokens,
recoveryAttempt: Number.isInteger(recoveryAttempt) && recoveryAttempt > 0 ? recoveryAttempt : 0,
};
}

export function emptyOutputFailureMessage(diagnostics = {}) {
switch (diagnostics.emptyReason) {
case 'output_limit':
return '[The model reached its configured response-token limit without producing visible text or a tool call, even after a recovery nudge. Reduce reasoning effort or choose another model/provider.]';
case 'reasoning_only':
return '[The latest model response contained reasoning but no visible answer or tool call after the earlier empty response. Reduce reasoning effort or choose another model/provider.]';
case 'content_filter':
return '[The provider filtered the latest response after an earlier empty response, so no visible text or tool call was returned. Revise the request or try another provider.]';
default:
return '[The provider returned an empty completion with no visible text or tool call twice. Retry later or choose another model/provider.]';
}
}
25 changes: 24 additions & 1 deletion src/chrome/src/agent/trace-export.js
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,24 @@ function renderLosslessRequest(messages, tools) {
function oneLine(t) { return String(t ?? '').replace(/\s+/g, ' ').trim(); }
function humanSize(n) { return n >= 1024 ? `${(n / 1024).toFixed(1)}kb` : `${n}b`; }

function renderEmptyModelResponse(data) {
const details = [
`reason=${oneLine(data.emptyReason || 'unknown')}`,
data.finishReason ? `finish=${oneLine(data.finishReason)}` : '',
Number.isInteger(data.outputTokens) ? `output=${data.outputTokens} tokens` : '',
Number.isInteger(data.reasoningTokens)
? `reasoning=${data.reasoningTokens} tokens`
: (Number.isInteger(data.reasoningChars) && data.reasoningChars > 0
? `reasoning=${data.reasoningChars} chars`
: (data.reasoningPresent === true ? 'reasoning=present' : '')),
Number.isInteger(data.requestedMaxTokens) ? `limit=${data.requestedMaxTokens} tokens` : '',
Number.isInteger(data.recoveryAttempt) ? `attempt=${data.recoveryAttempt}` : '',
`${Number.isInteger(data.contentChars) ? data.contentChars : 0} visible chars`,
`${Number.isInteger(data.toolCallCount) ? data.toolCallCount : 0} tool calls`,
].filter(Boolean).join(' 路 ');
return `- 馃 Empty model response: ${details}\n`;
}

function truncate(text, limit) {
const s = String(text ?? '');
if (s.length <= limit) return s;
Expand Down Expand Up @@ -274,7 +292,12 @@ export function tracesToMarkdown(runsWithEvents, {
md += `- 馃 Model request: ${Number(d.messageCount) || 0} messages 路 ${Number(d.toolsCount) || 0} tools${media ? ` 路 ${media}` : ''}${renderLocalWikipediaRag(d.localWikipediaRag)}${renderPromptProvenance(d.promptProvenance)}${d.lossless === true ? renderLosslessRequest(d.messages, d.tools) : ''}\n`;
} else if (ev.kind === 'llm_response') {
const content = String(d.content || '').trim();
if (!content) continue;
if (!content) {
if (!Array.isArray(d.toolCalls) || d.toolCalls.length === 0) {
md += renderEmptyModelResponse(d);
}
continue;
}
// Plan-before-Act runs record the planner call with phase:'planner'; keep
// it (derails often start in the plan) but label it and preserve its shape.
if (d.phase === 'planner') {
Expand Down
1 change: 1 addition & 0 deletions src/chrome/src/providers/openai.js
Original file line number Diff line number Diff line change
Expand Up @@ -978,6 +978,7 @@ export class OpenAICompatibleProvider extends BaseLLMProvider {
reasoningContent: message?.reasoning_content || message?.reasoning || '',
toolCalls: message?.tool_calls || null,
usage: data.usage || null,
finishReason: String(data?.choices?.[0]?.finish_reason || ''),
raw: data,
};
}
Expand Down
33 changes: 32 additions & 1 deletion src/chrome/src/trace/recorder.js
Original file line number Diff line number Diff line change
Expand Up @@ -665,7 +665,27 @@ export function recordLLMRequest(runId, step, payload, provenanceInput = null) {
});
}

export function recordLLMResponse(runId, step, { content, toolCalls, usage, latencyMs, model, phase, attempt, repair }) {
export function recordLLMResponse(runId, step, {
content,
toolCalls,
usage,
latencyMs,
model,
phase,
attempt,
repair,
empty,
emptyReason,
finishReason,
contentChars,
toolCallCount,
reasoningPresent,
reasoningChars,
reasoningTokens,
outputTokens,
requestedMaxTokens,
recoveryAttempt,
}) {
return _appendEvent(runId, 'llm_response', {
step,
content: content || null,
Expand All @@ -682,6 +702,17 @@ export function recordLLMResponse(runId, step, { content, toolCalls, usage, late
...(phase ? { phase } : {}),
...(Number.isInteger(attempt) ? { attempt } : {}),
...(repair === true ? { repair: true } : {}),
...(typeof empty === 'boolean' ? { empty } : {}),
...(emptyReason ? { emptyReason } : {}),
...(finishReason ? { finishReason } : {}),
...(Number.isInteger(contentChars) ? { contentChars } : {}),
...(Number.isInteger(toolCallCount) ? { toolCallCount } : {}),
...(typeof reasoningPresent === 'boolean' ? { reasoningPresent } : {}),
...(Number.isInteger(reasoningChars) ? { reasoningChars } : {}),
...(Number.isInteger(reasoningTokens) ? { reasoningTokens } : {}),
...(Number.isInteger(outputTokens) ? { outputTokens } : {}),
...(Number.isInteger(requestedMaxTokens) ? { requestedMaxTokens } : {}),
...(Number.isInteger(recoveryAttempt) && recoveryAttempt > 0 ? { recoveryAttempt } : {}),
});
}

Expand Down
18 changes: 17 additions & 1 deletion src/chrome/src/ui/traces.js
Original file line number Diff line number Diff line change
Expand Up @@ -525,9 +525,10 @@ function renderEvent(ev, shotCache, compact, objectUrls = new Set()) {
const usage = u ? `<span class="latency">${(u.prompt_tokens || 0).toLocaleString()} in / ${(u.completion_tokens || 0).toLocaleString()} out</span>` : '';
const lat = ev.data?.latencyMs != null ? `<span class="latency">${ev.data.latencyMs} ms</span>` : '';
const content = ev.data?.content;
const hasVisibleContent = typeof content === 'string' ? content.trim().length > 0 : !!content;
const toolCalls = ev.data?.toolCalls || [];
let body = '';
if (content) {
if (hasVisibleContent) {
body += `<div class="content-text">${escapeHtml(content)}</div>`;
}
if (toolCalls.length > 0) {
Expand All @@ -538,6 +539,21 @@ function renderEvent(ev, shotCache, compact, objectUrls = new Set()) {
}).join('');
body += `<div style="margin-top:6px;">${tcList}</div>`;
}
if (!hasVisibleContent && toolCalls.length === 0) {
const emptyDetails = [
`reason=${ev.data?.emptyReason || 'unknown'}`,
ev.data?.finishReason ? `finish=${ev.data.finishReason}` : '',
Number.isInteger(ev.data?.outputTokens) ? `output_tokens=${ev.data.outputTokens}` : '',
Number.isInteger(ev.data?.reasoningTokens)
? `reasoning_tokens=${ev.data.reasoningTokens}`
: (ev.data?.reasoningPresent === true ? 'reasoning_present=true' : ''),
Number.isInteger(ev.data?.requestedMaxTokens) ? `requested_max_tokens=${ev.data.requestedMaxTokens}` : '',
Number.isInteger(ev.data?.recoveryAttempt) ? `attempt=${ev.data.recoveryAttempt}` : '',
`content_chars=${Number.isInteger(ev.data?.contentChars) ? ev.data.contentChars : 0}`,
`tool_calls=${Number.isInteger(ev.data?.toolCallCount) ? ev.data.toolCallCount : 0}`,
].filter(Boolean).join(' 路 ');
body = `<div class="tool-args">EMPTY_RESPONSE 路 ${escapeHtml(emptyDetails)}</div>`;
}
return `
<div class="event llm_response">
<div class="event-head"><span class="kind">${escapeHtml(t('tr.event.llm_response'))}</span>${stepBadge}${usage}${lat}<span class="latency">${ts}</span></div>
Expand Down
Loading
Loading