feat(eval): enforce caller-supplied output schemas

packages/coding-agent/CHANGELOG.md

@@ -7,7 +7,29 @@
 ### Changed
 
 - Enhanced Anthropic credential and usage management to support organization-scoped accounts, including displaying organization names in /usage, /logout, omp token --list, and OAuth login success messages, resolving active-account matching for shared organizations, and deduplicating identities during migration.
+- `omp usage` and the in-session `/usage` view now show the Anthropic organization next to the account for org-scoped credentials (with `--redact` masking applied per part in the CLI, falling back to the org id when no display name is available), attribute "no usage data" rows per organization, and match the "in use by this session" marker by organization so only the active subscription is flagged. The OAuth login success message names the account and organization that was stored — a login landing on an unintended subscription is visible immediately.
+- `/logout` labels Anthropic accounts with their organization and marks only the credential of the active organization as active; `omp token --list` shows the organization next to each account. Two subscriptions sharing one email are distinguishable when selecting which to remove or mint a token for.
+- `omp auth-broker migrate --from-local` dedupes Anthropic OAuth identities per organization, so a Team seat already on the broker no longer blocks uploading the personal plan under the same email.
+- The status line invalidates its cached usage when the session rotates to a different Anthropic organization (previously the old subscription's quota could linger for the cache TTL), and `omp auth-gateway check` labels each credential with its organization so a failing row says which subscription needs re-login.
+- `omp usage` "no usage data" attribution is org-decisive whenever either the stored account or a report carries an organization: an org-less legacy credential whose own fetch failed is no longer hidden by an org-attributed sibling report sharing the same email.
+- Active-account matching for `/usage`, `/logout`, and `omp token --list` now treats a shared organization as a qualifier rather than a match: two Anthropic Team seats in one org (same org id, per-user pools) no longer flag each other's rows or reports as "in use by this session" — the base identity (account/email/project) is still required, with org-only sessions matching on the org alone.
+- `omp usage` "no usage data" coverage now requires the member's own identity within a shared organization: a sibling Team member's same-org report no longer counts as coverage for an account whose own report is missing, while an org-only account remains covered by any same-org report.
+- `omp auth-broker migrate --from-local` reruns now recognize an already-migrated org-only Anthropic row (login recovered neither email nor account) by its organization id instead of re-uploading it, which could overwrite the broker's newer refresh token with the stale local one.
+- Updated tangential agent forks to ignore parent session history and focus exclusively on the new request
+- Hardened `/tan` fork isolation: the clone's inherited todo list is cleared at fork (parent todo reminders no longer drag the tan back onto the parent's task), the fork notice warns that the parent is concurrently editing the same working directory, and the notice is re-injected after each compaction so the fork boundary survives summarization
+- Added visual markers in the transcript for elided tool calls that have no corresponding result
+- Updated status event log to prioritize the most recent entries in the display window
+- Updated the snapcompact shape preview transcript to use the compact scope format shown to models during compaction.
 
+### Removed
+
+- Removed the unreliable Bing and Yahoo HTML-scraping web search providers
+## [16.4.8] - 2026-07-12
+### Added
+
+- Added fail-closed validation for caller-supplied eval subagent schemas, with pre-dispatch schema checks, bounded correction attempts, exact final assembly validation, and typed schema-violation errors across eval runtimes; task-agent native schemas remain best effort ([#5278](https://github.com/can1357/oh-my-pi/issues/5278))
+
+## [16.4.8] - 2026-07-12
 ### Fixed
 
 - Fixed compatibility of GNU-flavored shell builtins (such as stat, date, sed, mktemp, tail, find, base64, and ln) when invoked with macOS/BSD-style arguments.

packages/coding-agent/src/eval/__tests__/agent-bridge.test.ts

@@ -4,7 +4,7 @@ import * as path from "node:path";
 import { TempDir } from "@oh-my-pi/pi-utils";
 import { Settings } from "../../config/settings";
 import { AgentProtocolHandler } from "../../internal-urls/agent-protocol";
-import { resetRegisteredArtifactDirsForTests } from "../../internal-urls/registry-helpers";
+import { artifactsDirsFromRegistry, resetRegisteredArtifactDirsForTests } from "../../internal-urls/registry-helpers";
 import type { PlanModeState } from "../../plan-mode/state";
 import { AgentRegistry } from "../../registry/agent-registry";
 import type { AgentSession } from "../../session/agent-session";
@@ -15,6 +15,7 @@ import * as isolationRunner from "../../task/isolation-runner";
 import { AgentOutputManager } from "../../task/output-manager";
 import type { AgentDefinition, AgentProgress, SingleResult } from "../../task/types";
 import type { ToolSession } from "../../tools";
+import type { SchemaViolationResult } from "../../tools/output-schema-validator";
 import { EVAL_AGENT_MAX_DEPTH, runEvalAgent } from "../agent-bridge";
 import { EVAL_TIMEOUT_PAUSE_OP, EVAL_TIMEOUT_RESUME_OP } from "../bridge-timeout";
 import { IdleTimeout } from "../idle-timeout";
@@ -309,6 +310,71 @@ describe("runEvalAgent", () => {
 		expect(firstOptions.modelOverride).toEqual(["p/override"]);
 		expect(secondOptions.outputSchema).toBeUndefined();
 		expect(secondOptions.outputSchemaOverridesAgent).toBeUndefined();
+		expect(secondOptions.schemaMode).toBeUndefined();
+		expect(secondOptions.preparedOutputSchema).toBeUndefined();
+	});
+
+	it("always prepares explicit schemas in strict mode", async () => {
+		mockAgents();
+		const schema = { type: "object", properties: { accepted: { type: "boolean" } }, required: ["accepted"] };
+		const runSpy = vi.spyOn(taskExecutor, "runSubprocess").mockImplementation(async options => singleResult(options));
+
+		await runEvalAgent({ prompt: "structured", schema }, { session: makeSession() });
+
+		const options = runSpy.mock.calls[0]?.[0];
+		if (!options) throw new Error("runSubprocess was not called");
+		expect(options.schemaMode).toBe("strict");
+		expect(options.preparedOutputSchema?.schemaMode).toBe("strict");
+	});
+
+	it("rejects invalid explicit schemas before dispatching", async () => {
+		mockAgents();
+		const runSpy = vi.spyOn(taskExecutor, "runSubprocess").mockImplementation(async options => singleResult(options));
+
+		await expect(runEvalAgent({ prompt: "invalid", schema: false }, { session: makeSession() })).rejects.toThrow(
+			"invalid schema",
+		);
+		expect(runSpy).not.toHaveBeenCalled();
+	});
+
+	it("preserves a valid structured result and exposes explicit-schema failures as typed bridge results", async () => {
+		mockAgents();
+		const schema = { type: "object", properties: { accepted: { type: "boolean" } }, required: ["accepted"] };
+		const violation: SchemaViolationResult = {
+			error: "schema_violation",
+			message: "result.data.accepted must be boolean",
+			missingRequired: [],
+			data: '{"accepted":"no"}',
+		};
+		const runSpy = vi.spyOn(taskExecutor, "runSubprocess");
+		runSpy.mockImplementationOnce(async options => singleResult(options, { output: '{"accepted":false}' }));
+		runSpy.mockImplementationOnce(async options => singleResult(options, { exitCode: 1, failure: violation }));
+
+		const success = await runEvalAgent({ prompt: "valid", schema }, { session: makeSession() });
+		const failure = await runEvalAgent({ prompt: "invalid", schema }, { session: makeSession() });
+
+		expect(success).toEqual({
+			text: '{"accepted":false}',
+			details: expect.objectContaining({ structured: true, agent: "task" }),
+		});
+		expect(failure).toMatchObject({
+			text: "ok",
+			details: { structured: true, agent: "task" },
+			schemaViolation: violation,
+		});
+	});
+
+	it("keeps unstructured subagent failures on the generic ToolError path", async () => {
+		mockAgents();
+		const runSpy = vi
+			.spyOn(taskExecutor, "runSubprocess")
+			.mockImplementation(async options => singleResult(options, { exitCode: 1, error: "subagent failed" }));
+
+		await expect(runEvalAgent({ prompt: "unstructured" }, { session: makeSession() })).rejects.toThrow(
+			"subagent failed",
+		);
+
+		expect(runSpy).toHaveBeenCalledTimes(1);
 	});
 
 	it("forces LSP off for bridge subagents even when task.enableLsp is on", async () => {
@@ -981,6 +1047,57 @@ describe("runEvalAgent isolation", () => {
 		expect(mergeSpy).toHaveBeenCalledTimes(1);
 	});
 
+	it("skips isolated apply but accounts for and removes strict-failure artifacts even with handle", async () => {
+		mockAgents();
+		mockIsolationContext();
+		const violation: SchemaViolationResult = {
+			error: "schema_violation",
+			message: "result.data.accepted must be boolean",
+			missingRequired: [],
+			data: '{"accepted":"no"}',
+		};
+		const usage: number[] = [];
+		const session = isolatedSession();
+		session.recordEvalSubagentUsage = output => usage.push(output);
+		let artifactsDir: string | undefined;
+		vi.spyOn(isolationRunner, "runIsolatedSubprocess").mockImplementation(async options => {
+			artifactsDir = options.baseOptions.artifactsDir;
+			if (!artifactsDir) throw new Error("artifactsDir missing");
+			await fs.writeFile(path.join(artifactsDir, "strict-failure.md"), "must be removed");
+			return singleResult(options.baseOptions, {
+				exitCode: 1,
+				output: "",
+				failure: violation,
+				usage: {
+					input: 0,
+					output: 17,
+					cacheRead: 0,
+					cacheWrite: 0,
+					totalTokens: 17,
+					cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+				},
+			});
+		});
+		const mergeSpy = vi.spyOn(isolationRunner, "mergeIsolatedChanges");
+
+		const result = await runEvalAgent(
+			{ prompt: "classify", schema: { type: "object" }, isolated: true, handle: true },
+			{ session },
+		);
+
+		expect(result.schemaViolation).toEqual(violation);
+		expect(mergeSpy).not.toHaveBeenCalled();
+		expect(usage).toEqual([17]);
+		if (!artifactsDir) throw new Error("strict failure did not allocate artifacts");
+		expect(
+			await fs.stat(artifactsDir).then(
+				() => true,
+				() => false,
+			),
+		).toBe(false);
+		expect(artifactsDirsFromRegistry()).not.toContain(artifactsDir);
+	});
+
 	it("preserves temp artifacts for non-isolated handle outputs", async () => {
 		mockAgents();
 		const rmSpy = vi.spyOn(fs, "rm").mockResolvedValue(undefined);

packages/coding-agent/src/eval/__tests__/julia-prelude.test.ts

@@ -63,4 +63,72 @@ nothing
 		// Frames are still present alongside the message.
 		expect(result.output).toContain("top-level scope");
 	}, 30_000);
+	it("forwards schema and isolation controls without exposing schema-mode controls", async () => {
+		using tempDir = TempDir.createSync("@omp-eval-julia-agent-");
+		const result = await executeJulia(
+			`
+global seen_agent_args = nothing
+function __omp_call_bridge(name::String, args::Dict{String, Any})
+    @assert name == "__agent__"
+    global seen_agent_args = copy(args)
+    if args["prompt"] == "invalid"
+        return Dict{String, Any}("schemaViolation" => Dict{String, Any}(
+            "message" => "result.data.ok must be boolean",
+            "data" => "{\\"ok\\":1}"
+        ))
+    end
+    return Dict{String, Any}(
+        "text" => "{\\"ok\\":true}",
+        "details" => Dict{String, Any}("id" => "julia-agent-1", "agent" => "task")
+    )
+end
+
+node = agent(
+    "valid";
+    schema=Dict("type" => "object"),
+    isolated=true,
+    apply=false,
+    merge=true,
+    handle=true
+)
+@assert node["data"]["ok"] === true
+@assert node["handle"] == "agent://julia-agent-1"
+@assert seen_agent_args["schema"]["type"] == "object"
+@assert seen_agent_args["isolated"] === true
+@assert seen_agent_args["apply"] === false
+@assert seen_agent_args["merge"] === true
+@assert seen_agent_args["handle"] === true
+@assert !haskey(seen_agent_args, "schemaMode")
+@assert !haskey(seen_agent_args, "schema_mode")
+
+try
+    agent("invalid"; schema=Dict("type" => "object"))
+    error("expected AgentSchemaViolationError")
+catch err
+    @assert err isa AgentSchemaViolationError
+    @assert err.code == "schema_violation"
+    @assert err.details["message"] == "result.data.ok must be boolean"
+end
+
+for legacy_keyword in (:schema_mode, :schemaMode)
+    try
+        agent("legacy"; legacy_keyword => "strict")
+        error("expected unsupported legacy schema keyword")
+    catch err
+        @assert err isa MethodError
+    end
+end
+println("AGENT_FORWARDING_OK")
+`,
+			{
+				cwd: tempDir.path(),
+				sessionId: `julia-prelude-agent:${crypto.randomUUID()}`,
+				kernelOwnerId: OWNER_ID,
+				reset: true,
+			},
+		);
+
+		expect(result.exitCode).toBe(0);
+		expect(result.output).toContain("AGENT_FORWARDING_OK");
+	}, 30_000);
 });

packages/coding-agent/src/eval/__tests__/prelude-agent.test.ts

@@ -104,4 +104,104 @@ describe("eval js agent() handle", () => {
 		expect(node.isolationSummary).toContain("/artifacts/iso-1.patch");
 		expect("branchName" in node).toBe(false);
 	});
+
+	it("preserves trailing object mappings and forwards schema", async () => {
+		let seenArgs: Record<string, unknown> | undefined;
+		const sandbox = loadPrelude(async (_name, args) => {
+			seenArgs = args as Record<string, unknown>;
+			return {
+				text: '{"accepted":false,"zero":0}',
+				details: { agent: "task", id: "strict-1", structured: true },
+			};
+		});
+
+		const output = await (sandbox.agent as AgentHelper)("classify", {
+			agent: "task",
+			model: ["p/fast"],
+			label: "Classify",
+			schema: { type: "object" },
+			isolated: true,
+			apply: false,
+			merge: true,
+			handle: false,
+		});
+
+		expect(seenArgs).toEqual({
+			prompt: "classify",
+			agent: "task",
+			model: ["p/fast"],
+			label: "Classify",
+			schema: { type: "object" },
+			isolated: true,
+			apply: false,
+			merge: true,
+			handle: false,
+		});
+		expect(output).toEqual({ accepted: false, zero: 0 });
+	});
+
+	it("keeps legacy positional isolation/apply/merge/handle slots", async () => {
+		let seenArgs: Record<string, unknown> | undefined;
+		const schema = { type: "object" };
+		const sandbox = loadPrelude(async (_name, args) => {
+			seenArgs = args as Record<string, unknown>;
+			return {
+				text: '{"ok":true}',
+				details: { agent: "reviewer", id: "positional-1", structured: true },
+			};
+		});
+
+		const output = await (sandbox.agent as (...args: unknown[]) => Promise<unknown>)(
+			"inspect",
+			"reviewer",
+			["p/fast"],
+			"Positionally",
+			schema,
+			true,
+			false,
+			true,
+			false,
+		);
+
+		expect(seenArgs).toEqual({
+			prompt: "inspect",
+			agent: "reviewer",
+			model: ["p/fast"],
+			label: "Positionally",
+			schema,
+			isolated: true,
+			apply: false,
+			merge: true,
+			handle: false,
+		});
+		expect(output).toEqual({ ok: true });
+	});
+
+	it("throws a machine-distinguishable error with violation details", async () => {
+		const violation = {
+			error: "schema_violation",
+			message: "result.data.answer must be string",
+			missingRequired: [],
+			data: '{"answer":7}',
+		};
+		let seenArgs: Record<string, unknown> | undefined;
+		const sandbox = loadPrelude(async (_name, args) => {
+			seenArgs = args as Record<string, unknown>;
+			return { schemaViolation: violation };
+		});
+
+		await expect(
+			(sandbox.agent as AgentHelper)("classify", { schema: { type: "object" }, handle: true }),
+		).rejects.toMatchObject({
+			name: "AgentSchemaViolationError",
+			code: "schema_violation",
+			message: violation.message,
+			details: violation,
+		});
+		expect(seenArgs).toEqual({
+			prompt: "classify",
+			schema: { type: "object" },
+			handle: true,
+		});
+	});
 });