From 513c9efbdbf9f73df042e5715de53bf9bf4c61d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=20=E4=B8=80=20M?= <202411103024@stu.sdjzu.edu.cn> Date: Mon, 31 Aug 2026 01:29:22 +0800 Subject: [PATCH 1/5] =?UTF-8?q?fix(core):=20UNKNOWN=5FTOOL=20=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E9=99=84=E5=B8=A6=E5=8F=AF=E7=94=A8=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E6=B8=85=E5=8D=95=E4=B8=8E=E6=9C=80=E8=BF=91=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E5=BB=BA=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 基线实测 FM1：模型幻觉顶层工具名（如 read_file）后，原报错只说 'not registered'，不给可用工具信息，导致 7+ 轮无效重试循环。 - 顶层调用失败时列出真实可见工具 + Code Mode 嵌套绑定，命中相近 名字时直接给出 tools.(...) 的正确调用方式 - Levenshtein<=2 + fuzzy>=60 双通道最近匹配，覆盖换位/漏分隔符拼写 - nested 作用域单独措辞，避免在沙箱内误导为顶层调用 验证：core 298 测试全绿；T4a 陷阱任务待重测 --- .../src/tools/runtime-unknown-tool.test.ts | 89 ++++++++++ packages/core/src/tools/runtime.ts | 159 +++++++++++++++++- 2 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 packages/core/src/tools/runtime-unknown-tool.test.ts diff --git a/packages/core/src/tools/runtime-unknown-tool.test.ts b/packages/core/src/tools/runtime-unknown-tool.test.ts new file mode 100644 index 00000000..3f587500 --- /dev/null +++ b/packages/core/src/tools/runtime-unknown-tool.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect } from "vitest"; +import { formatUnknownToolMessage } from "./runtime.js"; + +describe("formatUnknownToolMessage", () => { + const codeModeTools = { + name: "read_file", + scope: "top-level" as const, + availableTools: ["exec", "wait"], + nestedToolBindings: [ + "apply_patch", + "edit_file", + "list_directory", + "read_file", + "run_command", + ], + }; + + it("routes a hallucinated top-level call to its nested code-mode binding", () => { + const message = formatUnknownToolMessage(codeModeTools); + expect(message).toContain("Tool 'read_file' is not registered."); + expect(message).toContain("Available tools: exec, wait."); + expect(message).toContain( + "call exec and use tools.read_file(...) inside it", + ); + expect(message).not.toContain("Nested bindings:"); + }); + + it("lists all nested bindings when no close match exists", () => { + const message = formatUnknownToolMessage({ + ...codeModeTools, + name: "open_browser", + }); + expect(message).toContain("Tool 'open_browser' is not registered."); + expect(message).toContain("Nested bindings: apply_patch, edit_file"); + expect(message).toContain("run_command."); + expect(message).toContain("Do not retry this top-level call."); + }); + + it("suggests the closest direct tool when code mode is off", () => { + const message = formatUnknownToolMessage({ + name: "raed_file", + scope: "top-level", + availableTools: ["read_file", "edit_file", "run_command"], + }); + expect(message).toContain("Available tools: read_file, edit_file"); + expect(message).toContain("Did you mean 'read_file'?"); + expect(message).not.toContain("Code Mode"); + }); + + it("does not suggest garbage names", () => { + const message = formatUnknownToolMessage({ + name: "xkjzzz", + scope: "top-level", + availableTools: ["read_file", "run_command"], + }); + expect(message).not.toContain("Did you mean"); + }); + + it("uses nested-tool phrasing for calls from inside the sandbox", () => { + const message = formatUnknownToolMessage({ + name: "readfile", + scope: "nested", + availableTools: ["read_file", "edit_file"], + }); + expect(message).toContain("Available nested tools: read_file, edit_file."); + expect(message).toContain("Did you mean 'read_file'?"); + expect(message).not.toContain("Code Mode:"); + }); + + it("handles an empty tool list", () => { + const message = formatUnknownToolMessage({ + name: "read_file", + scope: "top-level", + availableTools: [], + }); + expect(message).toBe("Tool 'read_file' is not registered."); + }); + + it("caps very long tool lists", () => { + const many = Array.from({ length: 30 }, (_, index) => `tool_${index}`); + const message = formatUnknownToolMessage({ + name: "nope", + scope: "top-level", + availableTools: many, + }); + expect(message).toContain("tool_23 (+6 more)."); + expect(message).not.toContain("tool_24"); + }); +}); diff --git a/packages/core/src/tools/runtime.ts b/packages/core/src/tools/runtime.ts index 117b1895..3ca62a4b 100644 --- a/packages/core/src/tools/runtime.ts +++ b/packages/core/src/tools/runtime.ts @@ -275,6 +275,39 @@ export class ToolRuntime implements ToolRuntimeApi { this.baseSignal = signal; } + private unknownToolError( + name: string, + scope: "top-level" | "nested", + ): ToolExecutionResult { + if (scope === "nested") { + const nestedTools = this.codeModeEnabled + ? this.getCodeModeToolBindings().map((binding) => binding.identifier) + : [...this.nestedSpecsByName.keys()]; + return unknownToolResult(name, { + scope, + availableTools: nestedTools, + }); + } + + if (this.codeModeEnabled) { + return unknownToolResult(name, { + scope, + availableTools: this.listToolNames(), + nestedToolBindings: this.getCodeModeToolBindings().map( + (binding) => binding.identifier, + ), + }); + } + + return unknownToolResult(name, { + scope, + availableTools: [ + ...this.listToolNames(), + ...this.nestedSpecsByName.keys(), + ], + }); + } + async executeTool( name: string, rawArgs: string, @@ -284,7 +317,7 @@ export class ToolRuntime implements ToolRuntimeApi { if (internalName) { const visible = this.visibleSpecsByName.get(name); if (!visible) { - return unknownToolResult(name); + return this.unknownToolError(name, "top-level"); } return this.dispatchTool({ @@ -301,12 +334,12 @@ export class ToolRuntime implements ToolRuntimeApi { } if (this.codeModeEnabled) { - return unknownToolResult(name); + return this.unknownToolError(name, "top-level"); } const rawSpec = this.nestedSpecsByName.get(name); if (!rawSpec) { - return unknownToolResult(name); + return this.unknownToolError(name, "top-level"); } return this.dispatchTool({ @@ -330,12 +363,12 @@ export class ToolRuntime implements ToolRuntimeApi { ? name : this.nestedInternalNameByExternalName.get(name); if (!internalName) { - return unknownToolResult(name); + return this.unknownToolError(name, "nested"); } const spec = this.nestedSpecsByName.get(internalName); if (!spec) { - return unknownToolResult(name); + return this.unknownToolError(name, "nested"); } return this.dispatchTool({ @@ -770,17 +803,129 @@ function getExecutionErrorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } -function unknownToolResult(name: string): ToolExecutionResult { +export function formatUnknownToolMessage(input: { + name: string; + scope: "top-level" | "nested"; + availableTools: string[]; + nestedToolBindings?: string[]; +}): string { + const { name, scope, nestedToolBindings } = input; + const available = dedupeNames(input.availableTools).filter( + (item) => item !== name, + ); + const bindings = nestedToolBindings ? dedupeNames(nestedToolBindings) : []; + const nestedSuggestion = + scope === "top-level" && bindings.length > 0 + ? bestToolSuggestion(name, bindings) + : undefined; + const directSuggestion = + available.length > 0 ? bestToolSuggestion(name, available) : undefined; + + const parts: string[] = [`Tool '${name}' is not registered.`]; + if (available.length > 0) { + const label = + scope === "nested" ? "Available nested tools" : "Available tools"; + parts.push(`${label}: ${formatToolNameList(available)}`); + } + if (nestedSuggestion) { + parts.push( + `Code Mode: nested tool '${nestedSuggestion}' exists — call exec and use tools.${nestedSuggestion}(...) inside it instead of retrying this top-level call.`, + ); + } else if (scope === "top-level" && bindings.length > 0) { + parts.push( + `Code Mode: nested tools are callable only inside exec as tools.(...). Nested bindings: ${formatToolNameList(bindings)} Do not retry this top-level call.`, + ); + } + if (!nestedSuggestion && directSuggestion) { + parts.push(`Did you mean '${directSuggestion}'?`); + } + return parts.join(" "); +} + +function unknownToolResult( + name: string, + hints: { + scope: "top-level" | "nested"; + availableTools: string[]; + nestedToolBindings?: string[]; + }, +): ToolExecutionResult { return { ok: false, summary: `Unknown tool: ${name}`, error: { code: "UNKNOWN_TOOL", - message: `Tool '${name}' is not registered`, + message: formatUnknownToolMessage({ name, ...hints }), }, }; } +function bestToolSuggestion( + name: string, + candidates: string[], +): string | undefined { + let nearest: { name: string; distance: number } | undefined; + for (const candidate of candidates) { + const distance = levenshtein(name.toLowerCase(), candidate.toLowerCase()); + if (distance <= 2 && (!nearest || distance < nearest.distance)) { + nearest = { name: candidate, distance }; + } + } + if (nearest) { + return nearest.name; + } + + let best: { name: string; score: number } | undefined; + for (const candidate of candidates) { + const score = scoreFuzzyMatch(name, [{ text: candidate, weight: 1 }]); + if (score >= 60 && (!best || score > best.score)) { + best = { name: candidate, score }; + } + } + return best?.name; +} + +function levenshtein(left: string, right: string): number { + if (left === right) { + return 0; + } + if (left.length === 0) { + return right.length; + } + if (right.length === 0) { + return left.length; + } + + let previous = Array.from({ length: right.length + 1 }, (_, index) => index); + for (let leftIndex = 1; leftIndex <= left.length; leftIndex += 1) { + const current = [leftIndex]; + for (let rightIndex = 1; rightIndex <= right.length; rightIndex += 1) { + const substitution = + previous[rightIndex - 1] + + (left[leftIndex - 1] === right[rightIndex - 1] ? 0 : 1); + current[rightIndex] = Math.min( + previous[rightIndex] + 1, + current[rightIndex - 1] + 1, + substitution, + ); + } + previous = current; + } + return previous[right.length]; +} + +function dedupeNames(items: string[]): string[] { + return [...new Set(items.map((item) => item.trim()).filter(Boolean))]; +} + +function formatToolNameList(items: string[]): string { + const maxItems = 24; + if (items.length <= maxItems) { + return `${items.join(", ")}.`; + } + return `${items.slice(0, maxItems).join(", ")} (+${items.length - maxItems} more).`; +} + function inspectToolCall( spec: ToolSpec, args: unknown, From 8418cfa9a4cd60e029275af10fa3cad79d3c6d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=20=E4=B8=80=20M?= <202411103024@stu.sdjzu.edu.cn> Date: Mon, 31 Aug 2026 01:35:27 +0800 Subject: [PATCH 2/5] =?UTF-8?q?test(core):=20=E8=A1=A5=E5=85=85=20ToolRunt?= =?UTF-8?q?ime=20=E7=BA=A7=20UNKNOWN=5FTOOL=20=E6=8A=A5=E9=94=99=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E9=9B=86=E6=88=90=E6=96=AD=E8=A8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 验证顶层幻觉调用返回 exec 内 tools.(...) 路由指引、嵌套 幻觉调用返回真实嵌套工具清单与 Did-you-mean 建议 --- .../src/tools/runtime-unknown-tool.test.ts | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/packages/core/src/tools/runtime-unknown-tool.test.ts b/packages/core/src/tools/runtime-unknown-tool.test.ts index 3f587500..7fa16df9 100644 --- a/packages/core/src/tools/runtime-unknown-tool.test.ts +++ b/packages/core/src/tools/runtime-unknown-tool.test.ts @@ -1,5 +1,57 @@ import { describe, it, expect } from "vitest"; -import { formatUnknownToolMessage } from "./runtime.js"; +import { ToolRuntime, formatUnknownToolMessage } from "./runtime.js"; +import type { ToolExecutionContext, ToolSpec } from "@step-cli/protocol"; + +function makeSpec(name: string): ToolSpec { + return { + definition: { + type: "function", + function: { + name, + description: `${name} test tool`, + parameters: { type: "object", properties: {} }, + }, + }, + security: { risk: "read" }, + parseArgs: () => ({}), + execute: async () => ({ ok: true, summary: "ok" }), + }; +} + +const execContext: ToolExecutionContext = { + workspaceRoot: "/", + commandTimeoutMs: 1_000, + commandOutputLimit: 1_000, +}; + +describe("ToolRuntime unknown tool errors", () => { + it("routes a hallucinated top-level call to its nested code-mode binding", async () => { + const runtime = new ToolRuntime( + [makeSpec("exec"), makeSpec("wait"), makeSpec("read_file")], + execContext, + ); + const result = await runtime.executeTool("read_file", "{}"); + expect(result.ok).toBe(false); + expect(result.error?.code).toBe("UNKNOWN_TOOL"); + expect(result.error?.message).toContain("Available tools: exec, wait."); + expect(result.error?.message).toContain( + "call exec and use tools.read_file(...) inside it", + ); + }); + + it("reports unknown nested calls with the real nested tool list", async () => { + const runtime = new ToolRuntime( + [makeSpec("exec"), makeSpec("wait"), makeSpec("read_file")], + execContext, + ); + const result = await runtime.executeNestedTool("readfile", "{}"); + expect(result.ok).toBe(false); + expect(result.error?.message).toContain( + "Available nested tools: read_file.", + ); + expect(result.error?.message).toContain("Did you mean 'read_file'?"); + }); +}); describe("formatUnknownToolMessage", () => { const codeModeTools = { From 94b5cdebf74ec21f0eea6189dfae12bf9ab022d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=20=E4=B8=80=20M?= <202411103024@stu.sdjzu.edu.cn> Date: Mon, 31 Aug 2026 01:40:49 +0800 Subject: [PATCH 3/5] =?UTF-8?q?fix(core):=20=E5=B7=A5=E5=85=B7=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E6=8C=87=E7=BA=B9=E5=BD=92=E4=B8=80=E5=8C=96=E8=8D=92?= =?UTF-8?q?=E8=B0=AC=E5=A4=A7=E6=95=B0=EF=BC=8C=E5=B0=81=E5=A0=B5=E6=95=B0?= =?UTF-8?q?=E5=80=BC=E5=BE=AE=E8=B0=83=E9=80=83=E9=80=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 基线实测 FM2：模型失败后误诊为超时问题，指数级抬高 timeout_ms （2e10 → 2.4e37 → 3.6e76），每次只变一个数字就生成新指纹，完美 绕过 REPEATED_TOOL_CALL 拦截，600s 烧掉 1480 万 tokens。 - 指纹归一化时把 |数值| > 1e9 的参数折叠为占位符：合法参数 （timeout/offset/limit）远低于阈值不受影响，爆炸数值全部落进 同一指纹，被既有 repeatedToolCallLimit 正常拦截 - 拦截报错明确告知'只改数值参数也会被抑制'，直接矫正误诊回路 - 指纹逻辑迁移至 agent/tool-fingerprint.ts 并补 8 个单测 --- packages/core/src/agent/agent-loop.ts | 20 +---- .../core/src/agent/tool-fingerprint.test.ts | 78 +++++++++++++++++++ packages/core/src/agent/tool-fingerprint.ts | 68 ++++++++++++++++ 3 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 packages/core/src/agent/tool-fingerprint.test.ts create mode 100644 packages/core/src/agent/tool-fingerprint.ts diff --git a/packages/core/src/agent/agent-loop.ts b/packages/core/src/agent/agent-loop.ts index b9206fc3..b64f1e28 100644 --- a/packages/core/src/agent/agent-loop.ts +++ b/packages/core/src/agent/agent-loop.ts @@ -1,4 +1,4 @@ -import { createHash, randomUUID } from "node:crypto"; +import { randomUUID } from "node:crypto"; import type { ChatCompletionClient } from "../model-client.js"; import { isUnlimitedMaxSteps } from "../max-steps.js"; import type { @@ -52,6 +52,7 @@ import { type AgentWorkspaceMode, } from "./harness-context.js"; import { AgentStateMachine, type AgentStateSnapshot } from "./state-machine.js"; +import { createToolCallFingerprint } from "./tool-fingerprint.js"; export interface AgentLoopOptions { model: string; @@ -1200,7 +1201,7 @@ function blockedRepeatedToolCallResult( summary: `Suppressed repeated tool call '${toolName}' after ${limit} identical attempts`, error: { code: "REPEATED_TOOL_CALL", - message: `Attempt ${attempts} exceeded repeatedToolCallLimit=${limit}. Adjust arguments or produce a final response.`, + message: `Attempt ${attempts} exceeded repeatedToolCallLimit=${limit}. This call keeps being suppressed even when only numeric arguments change (e.g. a larger timeout). Diagnose why the identical call keeps failing, switch to a different approach, or produce a final response.`, }, data: { toolName, @@ -1260,21 +1261,6 @@ function toRunMetadata( }; } -function createToolCallFingerprint(toolName: string, rawArgs: string): string { - const normalizedArgs = normalizeToolArguments(rawArgs); - const hash = createHash("sha1").update(normalizedArgs).digest("hex"); - return `${toolName}:${hash}`; -} - -function normalizeToolArguments(rawArgs: string): string { - try { - const parsed = JSON.parse(rawArgs) as unknown; - return stableStringify(parsed); - } catch { - return rawArgs.replace(/\s+/g, " ").trim(); - } -} - function stableStringify(value: unknown): string { return JSON.stringify(sortRecursively(value)); } diff --git a/packages/core/src/agent/tool-fingerprint.test.ts b/packages/core/src/agent/tool-fingerprint.test.ts new file mode 100644 index 00000000..91a22aab --- /dev/null +++ b/packages/core/src/agent/tool-fingerprint.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from "vitest"; +import { + createToolCallFingerprint, + normalizeToolArguments, +} from "./tool-fingerprint.js"; + +describe("createToolCallFingerprint", () => { + it("collapses runaway numeric escalation into the same fingerprint", () => { + const base = { command: "cat missing.txt", timeout_ms: 20_000_000_000 }; + const second = { + command: "cat missing.txt", + timeout_ms: 2.4e37, + }; + const third = { timeout_ms: 3.6e76, command: "cat missing.txt" }; + expect(createToolCallFingerprint("exec", JSON.stringify(base))).toBe( + createToolCallFingerprint("exec", JSON.stringify(second)), + ); + expect(createToolCallFingerprint("exec", JSON.stringify(base))).toBe( + createToolCallFingerprint("exec", JSON.stringify(third)), + ); + }); + + it("keeps distinct legitimate calls distinct", () => { + const first = { path: "a.txt", offset: 600_000 }; + const second = { path: "a.txt", offset: 900_000 }; + expect( + createToolCallFingerprint("read_file", JSON.stringify(first)), + ).not.toBe(createToolCallFingerprint("read_file", JSON.stringify(second))); + }); + + it("still distinguishes different commands with identical huge timeouts", () => { + const first = { command: "ls", timeout_ms: 2e10 }; + const second = { command: "pwd", timeout_ms: 5e11 }; + expect(createToolCallFingerprint("exec", JSON.stringify(first))).not.toBe( + createToolCallFingerprint("exec", JSON.stringify(second)), + ); + }); + + it("includes the tool name in the fingerprint", () => { + const args = JSON.stringify({ path: "a.txt" }); + expect(createToolCallFingerprint("read_file", args)).not.toBe( + createToolCallFingerprint("edit_file", args), + ); + }); +}); + +describe("normalizeToolArguments", () => { + it("replaces huge finite numbers with a placeholder, key order independent", () => { + expect(normalizeToolArguments('{"b":2.4e37,"a":1}')).toBe( + normalizeToolArguments('{"a":1,"b":9.9e50}'), + ); + expect(normalizeToolArguments('{"timeout_ms":2e10}')).toContain( + "__HUGE_NUMBER__", + ); + }); + + it("keeps small and boundary numbers intact", () => { + expect(normalizeToolArguments('{"timeout_ms":600000,"n":-1.5}')).toBe( + '{"n":-1.5,"timeout_ms":600000}', + ); + expect(normalizeToolArguments('{"v":1000000000}')).toBe('{"v":1000000000}'); + expect(normalizeToolArguments('{"v":1000000001}')).toContain( + "__HUGE_NUMBER__", + ); + }); + + it("normalizes numbers nested in arrays and objects", () => { + const normalized = normalizeToolArguments( + '{"rows":[{"offset":1e12,"limit":10}]}', + ); + expect(normalized).toContain("__HUGE_NUMBER__"); + expect(normalized).toContain('"limit":10'); + }); + + it("falls back to whitespace collapsing for non-JSON arguments", () => { + expect(normalizeToolArguments(" run tests ")).toBe("run tests"); + }); +}); diff --git a/packages/core/src/agent/tool-fingerprint.ts b/packages/core/src/agent/tool-fingerprint.ts new file mode 100644 index 00000000..af30e733 --- /dev/null +++ b/packages/core/src/agent/tool-fingerprint.ts @@ -0,0 +1,68 @@ +import { createHash } from "node:crypto"; + +// Numeric arguments above this threshold collapse to a single placeholder in +// the fingerprint, so runaway escalation loops (e.g. timeout_ms 2e10 -> 2e37 -> +// 2e76 after each failure) count as repeated calls instead of fresh ones. +// Legitimate numeric arguments (timeouts, offsets, limits) stay far below it. +const HUGE_NUMBER_THRESHOLD = 1_000_000_000; +const HUGE_NUMBER_PLACEHOLDER = "__HUGE_NUMBER__"; + +export function createToolCallFingerprint( + toolName: string, + rawArgs: string, +): string { + const normalizedArgs = normalizeToolArguments(rawArgs); + const hash = createHash("sha1").update(normalizedArgs).digest("hex"); + return `${toolName}:${hash}`; +} + +export function normalizeToolArguments(rawArgs: string): string { + try { + const parsed = JSON.parse(rawArgs) as unknown; + return stableStringify(normalizeHugeNumbers(parsed)); + } catch { + return rawArgs.replace(/\s+/g, " ").trim(); + } +} + +function normalizeHugeNumbers(value: unknown): unknown { + if (typeof value === "number") { + return Number.isFinite(value) && Math.abs(value) > HUGE_NUMBER_THRESHOLD + ? HUGE_NUMBER_PLACEHOLDER + : value; + } + if (Array.isArray(value)) { + return value.map((entry) => normalizeHugeNumbers(entry)); + } + if (value && typeof value === "object") { + const normalized: Record = {}; + for (const [key, child] of Object.entries(value)) { + normalized[key] = normalizeHugeNumbers(child); + } + return normalized; + } + return value; +} + +function stableStringify(value: unknown): string { + return JSON.stringify(sortRecursively(value)); +} + +function sortRecursively(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((entry) => sortRecursively(entry)); + } + + if (value && typeof value === "object") { + const entries = Object.entries(value as Record).sort( + ([left], [right]) => left.localeCompare(right), + ); + const sorted: Record = {}; + for (const [key, child] of entries) { + sorted[key] = sortRecursively(child); + } + return sorted; + } + + return value; +} From 4329a198bc233a58710b748d3b3ea25438f1d504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=20=E4=B8=80=20M?= <202411103024@stu.sdjzu.edu.cn> Date: Mon, 31 Aug 2026 01:47:29 +0800 Subject: [PATCH 4/5] =?UTF-8?q?fix(core):=20exec=20=E6=B2=99=E7=9B=92?= =?UTF-8?q?=E5=8F=8D=E9=A6=88=E7=BB=93=E6=9E=84=E5=8C=96=EF=BC=8C=E6=B6=88?= =?UTF-8?q?=E9=99=A4=E5=B7=A5=E5=85=B7=E5=A4=B1=E8=B4=A5=E7=9A=84=E8=AF=AF?= =?UTF-8?q?=E8=AF=8A=E7=A9=BA=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 基线实测 FM3：脚本内工具失败只显示 ✗ 标记（丢弃 summary 里的 失败原因），且整体 ok:true + 模糊 Diagnostic，导致模型把命令 失败误诊为超时（FM2 灾难的直接诱因）。 - ✗ 行追加失败调用的真实 summary（错误码/stderr 摘要） - completed summary 如实计数失败调用（1 failed tool call） - 无返回值 Diagnostic 在有失败时明确排除超时误诊方向 - data 增加结构化 failedToolCalls 计数 - renderCellResult 导出并补 6 个单测 --- .../core/src/tools/code-mode/service.test.ts | 121 ++++++++++++++++++ packages/core/src/tools/code-mode/service.ts | 48 +++++-- 2 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 packages/core/src/tools/code-mode/service.test.ts diff --git a/packages/core/src/tools/code-mode/service.test.ts b/packages/core/src/tools/code-mode/service.test.ts new file mode 100644 index 00000000..1fd965b8 --- /dev/null +++ b/packages/core/src/tools/code-mode/service.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect } from "vitest"; +import { + renderCellResult, + type CellStatus, + type NestedToolCall, + type RunningCell, +} from "./service.js"; + +function makeCall(overrides?: Partial): NestedToolCall { + return { + toolName: "run_command", + identifier: "run_command", + ok: true, + summary: "Command completed", + ...overrides, + }; +} + +function makeCell(overrides?: Partial): RunningCell { + return { + id: "cell-test-1", + startedAt: Date.now(), + code: "const r = await tools.run_command({command: 'ls'});", + abortController: new AbortController(), + consoleLines: [], + status: "completed", + nestedCalls: [], + completion: Promise.resolve(), + ...overrides, + }; +} + +function render(status: CellStatus, cell: RunningCell) { + return renderCellResult({ cell, status, commandOutputLimit: 8000 }); +} + +describe("renderCellResult structured feedback", () => { + it("reports all-successful completions without failure noise", () => { + const cell = makeCell({ + nestedCalls: [makeCall(), makeCall({ toolName: "read_file" })], + }); + const result = render("completed", cell); + expect(result.ok).toBe(true); + expect(result.summary).toBe("Script completed · 2 tool calls"); + expect(result.data?.failedToolCalls).toBe(0); + expect(result.content).not.toContain("failed"); + }); + + it("surfaces per-call failure reasons that were previously hidden", () => { + const cell = makeCell({ + nestedCalls: [ + makeCall(), + makeCall({ + ok: false, + summary: + "Command failed with exit code 1: cat: missing.txt: No such file", + inputHint: "cat missing.txt", + }), + ], + }); + const result = render("completed", cell); + expect(result.ok).toBe(true); + expect(result.summary).toBe( + "Script completed · 2 tool calls, 1 failed tool call", + ); + expect(result.data?.failedToolCalls).toBe(1); + expect(result.content).toContain( + "✗ run_command cat missing.txt — Command failed with exit code 1", + ); + }); + + it("disambiguates no-return diagnostics when tool calls failed", () => { + const cell = makeCell({ + nestedCalls: [ + makeCall({ ok: false, summary: "Command failed with exit code 2" }), + ], + }); + const result = render("completed", cell); + expect(result.content).toContain("tool-level failures, not timeouts"); + expect(result.content).toContain( + "Script completed without a returned result.", + ); + }); + + it("keeps the plain no-return diagnostic when nothing failed", () => { + const cell = makeCell(); + const result = render("completed", cell); + expect(result.content).toContain( + "Script completed without a returned result.", + ); + expect(result.content).not.toContain("not timeouts"); + }); + + it("shows grouped failure reasons when calls exceed the per-call limit", () => { + const calls: NestedToolCall[] = []; + for (let index = 0; index < 6; index += 1) { + calls.push(makeCall({ inputHint: `attempt-${index}` })); + } + for (let index = 0; index < 4; index += 1) { + calls.push( + makeCall({ + ok: false, + summary: "Command failed with exit code 1", + inputHint: `failing-${index}`, + }), + ); + } + const cell = makeCell({ nestedCalls: calls }); + const result = render("completed", cell); + expect(result.content).toMatch(/6\/10 run_command ×10/); + expect(result.content).toContain("✗ Command failed with exit code 1"); + }); + + it("keeps failed script status reporting intact", () => { + const cell = makeCell({ errorText: "TypeError: boom" }); + const result = render("failed", cell); + expect(result.ok).toBe(false); + expect(result.summary).toBe("Script failed"); + expect(result.content).toContain("TypeError: boom"); + }); +}); diff --git a/packages/core/src/tools/code-mode/service.ts b/packages/core/src/tools/code-mode/service.ts index ff7f829d..023cc50b 100644 --- a/packages/core/src/tools/code-mode/service.ts +++ b/packages/core/src/tools/code-mode/service.ts @@ -17,9 +17,9 @@ const MAX_RENDER_CHARS = 120_000; const MIN_RENDER_CHARS = 200; const RUNTIME_BOOT_TIMEOUT_MS = 1_000; -type CellStatus = "running" | "completed" | "failed" | "terminated"; +export type CellStatus = "running" | "completed" | "failed" | "terminated"; -interface NestedToolCall { +export interface NestedToolCall { toolName: string; identifier: string; ok: boolean; @@ -59,7 +59,7 @@ interface CodeModeSandbox { TextDecoder: typeof TextDecoder; } -interface RunningCell { +export interface RunningCell { id: string; startedAt: number; code: string; @@ -547,16 +547,24 @@ function createAbortError(signal?: AbortSignal): Error { ); } -function renderCellResult(input: { +export function renderCellResult(input: { cell: RunningCell; status: CellStatus; commandOutputLimit: number; maxTokens?: number; -}): ToolExecutionResult { +}): ToolExecutionResult<{ + cell_id: string; + status: CellStatus; + running: boolean; + failedToolCalls: number; +}> { const prefixLines: string[] = []; const tailLines: string[] = []; let summary = ""; let ok = true; + const failedToolCalls = input.cell.nestedCalls.filter( + (call) => !call.ok, + ).length; switch (input.status) { case "running": { @@ -570,9 +578,18 @@ function renderCellResult(input: { } case "completed": { const toolCount = input.cell.nestedCalls.length; + const parts: string[] = []; + if (toolCount > 0) { + parts.push(`${toolCount} tool call${toolCount === 1 ? "" : "s"}`); + } + if (failedToolCalls > 0) { + parts.push( + `${failedToolCalls} failed tool call${failedToolCalls === 1 ? "" : "s"}`, + ); + } summary = - toolCount > 0 - ? `Script completed · ${toolCount} tool call${toolCount === 1 ? "" : "s"}` + parts.length > 0 + ? `Script completed · ${parts.join(", ")}` : "Script completed"; break; } @@ -608,7 +625,8 @@ function renderCellResult(input: { const hintStr = hint ? ` ${hint.length > 60 ? hint.slice(0, 57) + "..." : hint}` : ""; - tailLines.push(`${mark} ${call.toolName}${hintStr}`); + const reason = !call.ok && call.summary ? ` — ${call.summary}` : ""; + tailLines.push(`${mark} ${call.toolName}${hintStr}${reason}`); } } else { // Many calls: group by tool, show count and key details @@ -642,6 +660,14 @@ function renderCellResult(input: { const short = h.length > 68 ? h.slice(0, 65) + "..." : h; tailLines.push(` ${short}`); } + if (info.ok < info.count) { + const firstFailed = calls.find( + (call) => call.toolName === name && !call.ok, + ); + if (firstFailed?.summary) { + tailLines.push(` ✗ ${firstFailed.summary}`); + } + } if (info.hints.length < info.count && info.count > info.hints.length) { const more = info.count - info.hints.length; if (more > 0 && info.hints.length > 0) @@ -668,6 +694,11 @@ function renderCellResult(input: { if (input.status === "completed" && input.cell.result === undefined) { tailLines.push("Diagnostic:"); tailLines.push("Script completed without a returned result."); + if (failedToolCalls > 0) { + tailLines.push( + `${failedToolCalls} tool call(s) above failed — the script itself ran to completion, so these are tool-level failures, not timeouts. Read the ✗ lines for the actual error before retrying.`, + ); + } tailLines.push( "Return the final value directly from the top-level exec body if you need it in the model context.", ); @@ -746,6 +777,7 @@ function renderCellResult(input: { cell_id: input.cell.id, status: input.status, running: input.status === "running", + failedToolCalls, }, }; } From 899a2ef218e96cda5e0e478bcd4c82359384372d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Z=20=E4=B8=80=20M?= <202411103024@stu.sdjzu.edu.cn> Date: Mon, 31 Aug 2026 02:22:53 +0800 Subject: [PATCH 5/5] chore: reference tracking issue (#112) so link-check validates the updated PR body