diff --git a/docs/tools/task.md b/docs/tools/task.md index 044b64c934..6735865c5b 100644 --- a/docs/tools/task.md +++ b/docs/tools/task.md @@ -28,13 +28,15 @@ | Field | Type | Required | Description | | --- | --- | --- | --- | | `agent` | `string` | Yes | Exact agent name for every task item. Resolved at execution time through `discoverAgents(...)`. | -| `tasks` | `Array<{ id: string; description: string; assignment: string }>` | Yes | Batch of small, self-contained task items. `id` max length 48 in schema; duplicate ids are rejected case-insensitively at runtime. | +| `tasks` | `Array<{ id: string; description: string; assignment: string; tier?: "fast" \| "balanced" \| "strong" }>` | Yes | Batch of small, self-contained task items. `id` max length 48 in schema; duplicate ids are rejected case-insensitively at runtime. `tier` is advisory unless autorouting is enabled. | | `context` | `string` | No | Shared background prepended to every subagent system prompt. Trimmed before use. | | `schema` | `string` | No | JSON-encoded JTD schema. Overrides agent/session output schema when this mode allows task-level schemas. | | `isolated` | `boolean` | No | Only present when the tool is created with isolation enabled. Requests isolated execution for the whole batch. | `tasks[].description` is UI-only. `tasks[].assignment` is the actual per-task instruction. +`tasks[].tier` is inert while `task.autorouting.enabled` is `false`. When autorouting is active it selects the model chain for that item, an omitted `tier` routes as `balanced`, and the routed pin overrides the manual model chain. See [Autorouting](#autorouting). + ### Schema-free mode (`task.simple = "schema-free"`) Same as default, except `schema` is rejected by `validateTaskModeParams(...)` in `packages/coding-agent/src/task/index.ts`. @@ -151,6 +153,19 @@ Artifacts and side channels: - `planner` — read-only sequencing and acceptance criteria. - `critic` — read-only plan critique and actionability review. +## Autorouting + +Off by default. When `task.autorouting.enabled` is `true`, each task item is routed by its `tier` instead of the manual model chain. + +- Tier map source: `task.autorouting.tiers` (generated from `task.autorouting.setup`; provider order is seeded from `modelProviderOrder` and the model catalog). The removed `task.autorouting.preset` setting no longer participates in routing; enablement without usable tiers falls back to manual model resolution and reports a diagnostic. Contract in `packages/coding-agent/src/config/autorouting-contract.ts`. +- Selectors are exact `provider/modelId` strings with an optional `:minimal|low|medium|high|xhigh` suffix. Globs, bare model ids, and `pi/` aliases are rejected by the generated config schema. +- An omitted `tier` routes as `balanced`. A tier with no usable chain falls back to manual resolution for that item alone, with a bounded reason recorded. +- Preflight tries at most three unique candidates and only advances on transient failures observed before the run starts; there is no mid-run failover. Failed attempts run in staged sessions and attempt-scoped artifacts, so they leave no durable residue. +- Resolved routing evidence (skips, attempts, terminal outcome) is attached to the task result, receipt, renderer, and task-summary prompt. +- Setup: `/routing` opens the smart-routing panel (declare providers in priority order; chains are generated deterministically from the curated tier map and the model catalog, never from credentials). `/routing on|off|status` manages the toggle and prints effective chains. + +With autorouting disabled, model resolution is byte-for-byte unchanged. + ## Side Effects - Filesystem - Writes `context.md`, `.jsonl`, and `.md` under the session artifacts dir or a temp task dir. diff --git a/package.json b/package.json index 3cbf0c949d..ab3246d18c 100644 --- a/package.json +++ b/package.json @@ -110,7 +110,7 @@ "check:docker-context": "bun scripts/verify-docker-context.ts", "test:rs": "bun scripts/run-rs-task.ts test:rs", "check": "bun run --parallel check:ts check:rs", - "check:ts": "bun run check:tools && bun run check:publish-types && bun run check:node20-baseline && bun run check:public-sync && bun run check:schemas && bun run check:sdk-closure && bun run check:docker-context && bun run check:gjc-ui && bun run --workspaces --if-present check", + "check:ts": "bun run check:tools && bun run check:publish-types && bun run check:node20-baseline && bun run check:public-sync && bun run check:schemas && bun run check:sdk-closure && bun run check:docker-context && bun run check:gjc-ui && bun --cwd=packages/coding-agent run check:autorouting-map && bun run --workspaces --if-present check", "check:tools": "biome check . --no-errors-on-unmatched && tsc -p tsconfig.tools.json --noEmit", "check:publish-types": "bun scripts/ci-release-publish.ts --check-types", "check:node20-baseline": "bun scripts/check-node20-baseline.ts", @@ -130,7 +130,7 @@ "fix:tools": "biome check --write --unsafe --changed --no-errors-on-unmatched .", "fix:tools:all": "biome check --write --unsafe --no-errors-on-unmatched .", "fix:rs": "bun scripts/run-rs-task.ts fix:rs", - "ci:check:full": "bun run check:tools && bun run check:publish-types && bun run check:node20-baseline && bun run check:public-sync && bun run check:schemas && bun run check:sdk-skills && bun run check:docker-context && bun run check:gjc-ui && bun run --workspaces --if-present check", + "ci:check:full": "bun run check:tools && bun run check:publish-types && bun run check:node20-baseline && bun run check:public-sync && bun run check:schemas && bun run check:sdk-skills && bun run check:docker-context && bun run check:gjc-ui && bun --cwd=packages/coding-agent run check:autorouting-map && bun run --workspaces --if-present check", "ci:build:native": "bun scripts/ci-build-native.ts", "ci:test:full": "bun run test", "ci:test:smoke": "bun packages/coding-agent/src/cli.ts --version && bun packages/coding-agent/src/cli.ts --help && bun packages/coding-agent/src/cli.ts stats --help && bun packages/coding-agent/src/cli.ts --smoke-test", diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index d090abd074..384cc04b7a 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -6,10 +6,13 @@ ### Added - Added an SDK `automationTools` option for host-owned `browser` and `computer` implementations. External automation retains built-in activation and provenance, receives the normal abort signal, works independently of the default browser/platform gates, and fails closed on custom, extension, or MCP name collisions (#4809). +- Added opt-in Task autorouting that derives deterministic `fast`, `balanced`, and `strong` model tiers from declared provider priority. Tier generation is catalog-scoped and auth-independent, `/routing` opens the smart-routing setup panel, and bounded preflight evidence records candidate skips and acceptance outcomes. ### Changed - The SDK session index now self-repairs on append when it finds a corrupt suffix, instead of throwing `Cannot append to corrupt session index log` until an operator runs `gjc gc --repair-session-index`. The inline path reuses the same quarantine-backed repair (evidence preserved under `sessions/quarantine`, valid prefix republished), so a single poisoned row — e.g. a stale long-lived broker signing a `lifecycle_terminal` event against an outdated `indexSeq` — no longer wedges every subsequent session launch in that agent dir; the manual `gc` flag remains for non-converging damage. +- Breaking: removed `task.autorouting.preset` and the autorouting preset layer. Preset-only autorouting configuration is inactive until tiers are generated from smart-routing setup; public routing contracts no longer expose preset/source fields, and routing notes contain tier, fallback, and resume components only. +- Autorouting selectors now use one shared provider-qualified grammar and 256-character bound across runtime validation, generated tiers, routing evidence, and JSON Schema. Settings-derived tiers report provenance only when the provenance is valid; malformed provenance fails closed as hand-authored tiers. - Workflow handoffs out of autoresearch now transit properly: `/skill:deep-interview`, `/skill:ralplan`, and `/skill:ultragoal` chain from any live autoresearch phase (`intake`/`research`/`verdict`), and ralplan's `final` phase chains via its manifest terminal states; ralplan/ultragoal live phases still require the explicit write-to-handoff step. `gjc autoresearch clear` remains the finalize-only exit. - Breaking: autoresearch spec intake is now the explicit `intake` verb (`gjc autoresearch intake --spec `; the bare `--spec` flag form still works). The ambiguous `handoff` verb token is rejected with disambiguation hints instead of silently becoming a cold-intake goal — workflow handoff lives at `/skill:` (or `gjc state autoresearch handoff --to `). - Subagents now start correctly on Bun 1.4 when extension tools provide custom renderers. `RegisteredToolAdapter` installs its renderer adapters before proxying the definition, avoiding a write through the proxy's getter-only property while preserving the no-renderer fallback path. @@ -19,6 +22,8 @@ - SDK `goal.list/get` no longer reports `resource_gone` when an in-flight session's live goal projection is temporarily unavailable. It recovers the latest authoritative goal mode state from the current session branch after runtime recreation or session replacement, while goal-less sessions return an explicit `no_active_goal` diagnostic instead of being confused with snapshot-store loss (#4824). - `/logout` (and `gjc accounts logout`) now remove stored API-key credentials, not just OAuth rows. The interactive logout and the CLI only enumerated `oauth` inventory, so API-key logins (OpenCode Go/Zen, Cursor, Venice, DeepSeek, …) were rejected with `API-key credentials are not managed here`; both paths now remove stored credentials of either kind. +- Sanitized autorouting selectors and routing-summary attributes before persistence and `noEscape` prompt interpolation, including control characters and Unicode line separators. Generated tier scope now retains multimodal text-capable models while excluding only image-generation-only catalog entries. +- Autorouting preflight now uses a staged publication when durable session/artifact authority exists and an explicit artifact-only acceptance path when it does not, so a valid synchronous Task without a child session file is not rejected at the acceptance fence. - Queued SDK prompts now retain their dispatch-time ownership across selection fences instead of reclassifying from a contradictory later streaming snapshot. Fresh promotion, earlier follow-up ordering, and terminal-abort cancellation are reachable again, while `/btw` test fixtures now model the user-drainable queue count required by the current empty-submit contract. - Coordinator stop and idle-reap now initialize canonical namespace state before reading durable deletion recovery. Fresh or upgraded projection-only sessions were misreported as `state_corrupt` before the broker close was attempted, and completed deletion receipts were excluded from the idempotent missing-session lookup; both paths now preserve strict malformed-state rejection while allowing safe cleanup and replay. - Fixed provider safety-stop classification being lost before session persistence (#4777). The managed provider-envelope boundary now preserves only the allowlisted `provider_safety_stop` kind, typed safety stops remain terminal even with transport facts on a multi-model fallback chain, and the regression e2e test is selected both by the focused affected-path route and exactly one normal coding-agent shard. diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 8bca4d87f4..38abedc41d 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -33,6 +33,7 @@ "check": "biome check . && bun run check:types", "check:runtime": "bun run generate-hotkeys-docs --check && bun run verify:sdk-canonicalization", "check:types": "tsc -p tsconfig.json --noEmit", + "check:autorouting-map": "bun scripts/check-autorouting-tier-map.ts", "lint": "biome lint .", "test": "bun test", "generate-hotkeys-docs": "bun scripts/generate-hotkeys-docs.ts", @@ -149,6 +150,8 @@ "./sdk/host/control/runtime-gate.js": null, "./sdk/host/control/runtime-gate": null, "./sdk/host/control/runtime-gate/*": null, + "./sdk/host/internal-autorouting-state.js": null, + "./sdk/host/internal-autorouting-state": null, "./sdk/lifecycle/broker-client.js": null, "./sdk/lifecycle/broker-client": null, "./sdk/lifecycle/broker-client/*": null, diff --git a/packages/coding-agent/scripts/check-autorouting-tier-map.ts b/packages/coding-agent/scripts/check-autorouting-tier-map.ts new file mode 100644 index 0000000000..f39dffb7e7 --- /dev/null +++ b/packages/coding-agent/scripts/check-autorouting-tier-map.ts @@ -0,0 +1,183 @@ +#!/usr/bin/env bun + +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { RETIRED_MODEL_KEYS } from "../../ai/src/model-retirements"; +import { isValidAutoroutingSelector } from "../src/config/autorouting-contract"; +import { + CURATED_TIER_LABELS, + type CuratedTierLabels, + TIER_MAP_SKIP_LIST, + type TierMapSkipList, +} from "../src/config/autorouting-tier-map"; + +export const BASELINE_SKIP_RATIONALE = "pre-feature baseline; not yet curated"; +const DISCOVERY_ONLY_PROVIDERS = new Set(["ollama", "vllm"]); +const RETIRED_KEYS = new Set(RETIRED_MODEL_KEYS); + +export type CommittedCatalogModel = { + id?: unknown; + provider?: unknown; + output?: unknown; + [key: string]: unknown; +}; + +export type CommittedCatalog = Record>; + +export type AutoroutingTierMapGateReport = { + inScopeKeys: string[]; + labeledKeys: string[]; + skippedKeys: string[]; + unlabeledKeys: string[]; + invalidLabelKeys: string[]; + outOfScopeLabelKeys: string[]; + invalidSkipKeys: string[]; + staleSkipKeys: string[]; + baselineSkipCount: number; +}; + +function modelKey(provider: string, id: string): string { + return `${provider}/${id}`; +} + +function isImageGenerationOnly(model: CommittedCatalogModel): boolean { + return Array.isArray(model.output) && model.output.length > 0 && model.output.every(output => output === "image"); +} + +export function isInAutoroutingGateScope(provider: string, id: string, model: CommittedCatalogModel): boolean { + const key = modelKey(provider, id); + return ( + isValidAutoroutingSelector(key) && + !RETIRED_KEYS.has(key) && + !DISCOVERY_ONLY_PROVIDERS.has(provider) && + !isImageGenerationOnly(model) + ); +} + +export function committedCatalogKeys(catalog: CommittedCatalog): string[] { + const keys: string[] = []; + for (const [provider, models] of Object.entries(catalog)) { + if (models === null || typeof models !== "object" || Array.isArray(models)) continue; + for (const [id, model] of Object.entries(models)) { + if (isInAutoroutingGateScope(provider, id, model)) keys.push(modelKey(provider, id)); + } + } + return keys.sort((left, right) => left.localeCompare(right)); +} + +export function getAutoroutingTierMapGateReport( + catalog: CommittedCatalog, + labels: CuratedTierLabels = CURATED_TIER_LABELS, + skips: TierMapSkipList = TIER_MAP_SKIP_LIST, +): AutoroutingTierMapGateReport { + const catalogKeys = new Set(committedCatalogKeys(catalog)); + const labeledKeys = Object.keys(labels).sort((left, right) => left.localeCompare(right)); + const skippedKeys = Object.keys(skips).sort((left, right) => left.localeCompare(right)); + const invalidLabelKeys = labeledKeys.filter(key => !catalogKeys.has(key)); + const outOfScopeLabelKeys = labeledKeys.filter(key => { + const separator = key.indexOf("/"); + const provider = separator > 0 ? key.slice(0, separator) : key; + const id = separator > 0 ? key.slice(separator + 1) : ""; + const rawModel = catalog[provider]?.[id]; + return rawModel !== undefined && !isInAutoroutingGateScope(provider, id, rawModel); + }); + const unlabeledKeys = [...catalogKeys].filter(key => !Object.hasOwn(labels, key) && !Object.hasOwn(skips, key)); + const inScopeKeys = [...catalogKeys]; + const inScopeSkipped = skippedKeys.filter(key => catalogKeys.has(key)); + const invalidSkipKeys = skippedKeys.filter(key => { + const entry = (skips as Record)[key]; + const rationale = entry?.rationale; + return typeof rationale !== "string" || rationale.trim().length === 0 || !isValidAutoroutingSelector(key); + }); + const staleSkipKeys = skippedKeys.filter(key => !catalogKeys.has(key) && !invalidSkipKeys.includes(key)); + // A key that is both labeled and skipped would let a curated tier assignment hide + // behind a skip rationale; it is an invalid skip, not a stale one. + const bothLabeledAndSkipped = skippedKeys.filter(key => Object.hasOwn(labels, key)); + invalidSkipKeys.push(...bothLabeledAndSkipped.filter(key => !invalidSkipKeys.includes(key))); + return { + inScopeKeys, + labeledKeys, + skippedKeys: inScopeSkipped, + unlabeledKeys: unlabeledKeys.sort((left, right) => left.localeCompare(right)), + invalidLabelKeys, + outOfScopeLabelKeys, + invalidSkipKeys, + staleSkipKeys, + baselineSkipCount: inScopeSkipped.filter( + key => (skips as Record)[key]?.baseline === true, + ).length, + }; +} + +export function findUnlabeledAutoroutingKeys( + catalog: CommittedCatalog, + labels: CuratedTierLabels = CURATED_TIER_LABELS, + skips: TierMapSkipList = TIER_MAP_SKIP_LIST, +): string[] { + return getAutoroutingTierMapGateReport(catalog, labels, skips).unlabeledKeys; +} + +export type AutoroutingTierMapGateResult = { + ok: boolean; + report: AutoroutingTierMapGateReport; +}; + +export function checkAutoroutingTierMap( + catalog: CommittedCatalog, + labels: CuratedTierLabels = CURATED_TIER_LABELS, + skips: TierMapSkipList = TIER_MAP_SKIP_LIST, +): AutoroutingTierMapGateResult { + const report = getAutoroutingTierMapGateReport(catalog, labels, skips); + const ok = + report.unlabeledKeys.length === 0 && + report.invalidLabelKeys.length === 0 && + report.outOfScopeLabelKeys.length === 0 && + report.invalidSkipKeys.length === 0 && + report.staleSkipKeys.length === 0; + return { ok, report }; +} + +export async function loadCommittedCatalog( + repoRoot = path.resolve(import.meta.dir, "../../.."), +): Promise { + const catalogPath = path.join(repoRoot, "packages/ai/src/models.json"); + return JSON.parse(await fs.readFile(catalogPath, "utf8")) as CommittedCatalog; +} + +export async function runAutoroutingTierMapGate(repoRoot?: string): Promise { + return checkAutoroutingTierMap(await loadCommittedCatalog(repoRoot)); +} + +export const checkTierMap = checkAutoroutingTierMap; +export const findUnlabeledKeys = findUnlabeledAutoroutingKeys; +export const runTierMapGate = runAutoroutingTierMapGate; + +function printFailure(report: AutoroutingTierMapGateReport): void { + const failures = [ + ...new Set([ + ...report.unlabeledKeys, + ...report.invalidLabelKeys, + ...report.outOfScopeLabelKeys, + ...report.invalidSkipKeys, + ...report.staleSkipKeys, + ]), + ].sort((a, b) => a.localeCompare(b)); + console.error("Autorouting tier-map gate failed."); + console.error("Offending provider/model-id keys:"); + for (const key of failures) console.error(`- ${key}`); + console.error( + "Add each new key to CURATED_TIER_LABELS with reviewed tier/rank data, or add it to TIER_MAP_SKIP_LIST with a non-empty rationale; remove skip entries whose key left the catalog or the selector grammar.", + ); +} + +if (import.meta.main) { + const result = await runAutoroutingTierMapGate(); + if (!result.ok) { + printFailure(result.report); + process.exitCode = 1; + } else { + console.log( + `Autorouting tier-map gate passed: ${result.report.inScopeKeys.length} in-scope keys; ${result.report.baselineSkipCount} baseline skips.`, + ); + } +} diff --git a/packages/coding-agent/scripts/generate-sdk-operation-inventory.ts b/packages/coding-agent/scripts/generate-sdk-operation-inventory.ts index 12dd0a0235..e2afd3fa50 100644 --- a/packages/coding-agent/scripts/generate-sdk-operation-inventory.ts +++ b/packages/coding-agent/scripts/generate-sdk-operation-inventory.ts @@ -17,6 +17,8 @@ const LOCKED_EXCLUSIONS: Readonly> = { "internal terminal-abort bus seam, threaded via terminalAbortSeams; not a user-facing SDK control seam", "agent_session:abortPromptAndWaitWithTerminal": "internal terminal-abort fencing seam, threaded via terminalAbortSeams; not a user-facing SDK control seam", + "slash_command:routing": + "visual/local-only autorouting settings toggle and smart-routing panel entry, not a user-facing SDK control seam", "slash_command:settings": "visual/local-only command, not a user-facing SDK control seam", "slash_command:theme": "visual/local-only command, not a user-facing SDK control seam", "slash_command:copy": "visual/local-only command, not a user-facing SDK control seam", diff --git a/packages/coding-agent/scripts/generate-tool-catalog.ts b/packages/coding-agent/scripts/generate-tool-catalog.ts index 31c560e918..62c4d3d4a8 100644 --- a/packages/coding-agent/scripts/generate-tool-catalog.ts +++ b/packages/coding-agent/scripts/generate-tool-catalog.ts @@ -101,6 +101,8 @@ function makeSettings() { return {}; }, getNotificationSettingsSnapshot: () => ({ enabled: false, telegram: {}, discord: {}, slack: {} }), + // The catalog documents the default configuration, where autorouting is off. + getEffectiveAutorouting: () => ({ active: false }), }; } diff --git a/packages/coding-agent/src/cli/config-cli.ts b/packages/coding-agent/src/cli/config-cli.ts index 88a3eb10a2..e57e9fedbe 100644 --- a/packages/coding-agent/src/cli/config-cli.ts +++ b/packages/coding-agent/src/cli/config-cli.ts @@ -11,6 +11,11 @@ import { APP_NAME, getAgentDir } from "@gajae-code/utils"; import { YAML } from "bun"; import chalk from "chalk"; import { AtomicYamlReplaceError, AtomicYamlRetargetError } from "../config/atomic-yaml-patch"; +import { + validateAutoroutingLocal, + validateAutoroutingProvenance, + validateAutoroutingSetup, +} from "../config/autorouting-contract"; import { resolveModelProfileName } from "../config/model-profile-contract"; import { mergeModelProfiles } from "../config/model-profiles"; import { ModelsConfigFile } from "../config/model-registry"; @@ -208,7 +213,10 @@ function getTypeDisplay(def: CliSettingDef): string { case "array": return "(array)"; case "record": + case "constrained-record": return "(record)"; + case "optional-object": + return "(object)"; default: return "(string)"; } @@ -264,7 +272,9 @@ function parseAndSetValue(path: SettingPath, rawValue: string): void { parsedValue = parsed; break; } - case "record": { + case "record": + case "constrained-record": + case "optional-object": { let parsed: unknown; try { parsed = JSON.parse(trimmed); @@ -280,6 +290,17 @@ function parseAndSetValue(path: SettingPath, rawValue: string): void { default: parsedValue = trimmed; } + const issues = + path === "task.autorouting.tiers" + ? validateAutoroutingLocal({ tiers: parsedValue }) + : path === "task.autorouting.setup" + ? validateAutoroutingSetup(parsedValue) + : path === "task.autorouting.provenance" + ? validateAutoroutingProvenance(parsedValue) + : []; + if (issues.length > 0) { + throw new Error(`Invalid value for ${path}: ${issues.map(issue => `${issue.path}: ${issue.detail}`).join("; ")}`); + } settings.set(path, parsedValue as SettingValue); } @@ -581,6 +602,9 @@ function matchesSettingType(path: SettingPath, value: unknown): boolean { case "array": return Array.isArray(value); case "record": + case "constrained-record": + return value !== null && typeof value === "object" && !Array.isArray(value); + case "optional-object": return value !== null && typeof value === "object" && !Array.isArray(value); } } diff --git a/packages/coding-agent/src/config/autorouting-contract.ts b/packages/coding-agent/src/config/autorouting-contract.ts new file mode 100644 index 0000000000..c4d44e8320 --- /dev/null +++ b/packages/coding-agent/src/config/autorouting-contract.ts @@ -0,0 +1,474 @@ +import { createHash } from "node:crypto"; + +/** + * Dependency-free autorouting vocabulary and settings validators. + * + * This module deliberately does not import Settings, task code, or model + * profiles. It is the shared contract used by settings and (later) routing + * policy code. + */ + +export const AUTOROUTING_TIERS = ["fast", "balanced", "strong"] as const; +export type AutoroutingTier = (typeof AUTOROUTING_TIERS)[number]; +export const DEFAULT_AUTOROUTING_TIER: AutoroutingTier = "balanced"; + +/** + * The single wording for "autorouting is switched on but cannot route". + * + * One constant so every delivery surface (interactive, print, and the SDK/ACP + * host) reports the identical sentence and cannot drift apart. Bounded and free + * of interpolation so it is always safe to render. + */ +export const AUTOROUTING_INACTIVE_WARNING = + "Autorouting is enabled but has no usable tier chains; Task items fall back to manual model resolution. Run /routing to inspect, or /model \u2192 smart routing to generate tiers."; + +/** The normalized tier map consumed by routing policy. */ +export type TierMap = Partial>; + +/** The permissive input shape accepted by the settings surface. */ +export type AutoroutingTierMapInput = Partial>; + +export type AutoroutingSetup = { + schema: 1; + providers: string[]; + models?: string[]; +}; + +/** Fingerprints describing the generated autorouting declaration and materialized tiers. */ +export type AutoroutingProvenance = { + schema: 1; + source: { + catalogFingerprint: string; + mapFingerprint: string; + generatorVersion: number; + }; + declarationFingerprint: string; + tiersFingerprint: string; +}; + +/** Hard upper bound shared with the routing-evidence invariant in task/types.ts. */ +export const AUTOROUTING_SELECTOR_MAX_LENGTH = 256; + +/** The exact selector grammar published by the generated config schema. */ +export const AUTOROUTING_SELECTOR_PATTERN = + "^[^/\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+\\/[^\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+(?::(?:minimal|low|medium|high|xhigh))?$"; + +export const AUTOROUTING_SELECTOR_DESCRIPTION = + "provider/modelId with an optional valid thinking suffix (:minimal|low|medium|high|xhigh), no globs, no bare model ids, no pi/ role aliases."; + +export type AutoroutingReasonCode = + | "tier_unmatched" + | "tier_missing_in_map" + | "config_invalid" + | "map_absent" + | "selector_not_provider_qualified" + | "auth_substituted" + | "assistant_model_mismatch" + | "provider_disabled" + | "snapshot_missing" + | "credential_unavailable" + | "preflight_spawn_failed" + | "preflight_exhausted"; + +export type AutoroutingLocalIssue = { + path: string; + code: AutoroutingReasonCode; + /** Alias retained for callers that describe diagnostics as reasons. */ + reason: AutoroutingReasonCode; + detail: string; +}; + +export type AutoroutingEffectiveIssue = { + code: Extract; + /** Alias retained for callers that describe diagnostics as reasons. */ + reason: Extract; + detail: string; +}; + +export type AutoroutingEffective = + | { active: true; map: TierMap } + | { active: false; issue?: AutoroutingEffectiveIssue }; + +function issue(path: string, code: AutoroutingReasonCode, detail: string): AutoroutingLocalIssue { + return { path, code, reason: code, detail }; +} + +function effectiveIssue( + code: Extract, + detail: string, +): AutoroutingEffectiveIssue { + return { code, reason: code, detail }; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +/** + * Validate one provider-qualified selector. A selector has one provider + * segment, a non-empty model remainder, and may carry one supported thinking + * suffix. Model ids may themselves contain slashes; the provider is always + * the segment before the first slash. + */ +export function isValidAutoroutingSelector(value: unknown): value is string { + if (typeof value !== "string" || value.length === 0 || value.trim() !== value) return false; + if (value.length > AUTOROUTING_SELECTOR_MAX_LENGTH) return false; + if (/[*?[]/.test(value)) return false; + if (!new RegExp(AUTOROUTING_SELECTOR_PATTERN).test(value)) return false; + const separator = value.indexOf("/"); + if (separator <= 0) return false; + const provider = value.slice(0, separator); + return provider.toLowerCase() !== "pi"; +} + +export function normalizeTierMap(value: unknown): TierMap { + if (!isRecord(value)) return {}; + const normalized: TierMap = {}; + for (const tier of AUTOROUTING_TIERS) { + const raw = value[tier]; + const selectors = typeof raw === "string" ? [raw] : Array.isArray(raw) ? raw : []; + const usable = selectors.filter(isValidAutoroutingSelector); + if (usable.length > 0) normalized[tier] = [...usable]; + } + return normalized; +} + +/** True when at least one known tier contains one grammatically valid selector. */ +export function isMeaningfulTierMap(value: unknown): value is TierMap { + return Object.values(normalizeTierMap(value)).some(selectors => selectors.length > 0); +} + +function validateSelectorValue(path: string, value: unknown, issues: AutoroutingLocalIssue[]): void { + const selectors = typeof value === "string" ? [value] : Array.isArray(value) ? value : null; + if (!selectors || selectors.length === 0 || selectors.some(selector => typeof selector !== "string")) { + issues.push(issue(path, "config_invalid", "Expected a non-empty selector string or array of selector strings.")); + return; + } + for (let index = 0; index < selectors.length; index++) { + if (!isValidAutoroutingSelector(selectors[index])) { + issues.push( + issue( + Array.isArray(value) ? `${path}.${index}` : path, + "selector_not_provider_qualified", + `Expected ${AUTOROUTING_SELECTOR_DESCRIPTION}`, + ), + ); + } + } +} + +/** Validate the typed auto-setup declaration without consulting the model catalog. */ +export function validateAutoroutingSetup(value: unknown): AutoroutingLocalIssue[] { + const issues: AutoroutingLocalIssue[] = []; + if (!isRecord(value)) { + issues.push(issue("", "config_invalid", "Expected an autorouting setup object.")); + return issues; + } + for (const key of Object.keys(value)) { + if (key !== "schema" && key !== "providers" && key !== "models") { + issues.push(issue(key, "config_invalid", "Unknown autorouting setup key.")); + } + } + if (value.schema !== 1) issues.push(issue("schema", "config_invalid", "Expected schema version 1.")); + if (!Array.isArray(value.providers)) { + issues.push(issue("providers", "config_invalid", "Expected a non-empty array of provider names.")); + } else { + if (value.providers.length === 0) { + issues.push(issue("providers", "config_invalid", "Expected a non-empty array of provider names.")); + } + const seen = new Set(); + for (let index = 0; index < value.providers.length; index++) { + const provider = value.providers[index]; + if (typeof provider !== "string" || provider.length === 0 || provider.trim() !== provider) { + issues.push(issue(`providers.${index}`, "config_invalid", "Expected a non-empty provider name.")); + continue; + } + if (seen.has(provider)) { + issues.push(issue(`providers.${index}`, "config_invalid", "Provider declarations must be unique.")); + continue; + } + seen.add(provider); + } + } + if (value.models !== undefined) { + if (!Array.isArray(value.models)) { + issues.push(issue("models", "config_invalid", "Expected an array of provider-qualified model selectors.")); + } else { + for (let index = 0; index < value.models.length; index++) { + if (!isValidAutoroutingSelector(value.models[index])) { + issues.push( + issue( + `models.${index}`, + "selector_not_provider_qualified", + `Expected ${AUTOROUTING_SELECTOR_DESCRIPTION}`, + ), + ); + } + } + } + } + return issues; +} + +const FINGERPRINT_PATTERN = /^[0-9a-f]{64}$/u; + +function validateFingerprint(path: string, value: unknown, issues: AutoroutingLocalIssue[]): void { + if (typeof value !== "string" || !FINGERPRINT_PATTERN.test(value)) { + issues.push(issue(path, "config_invalid", "Expected a lowercase 64-character SHA-256 fingerprint.")); + } +} + +/** Validate generated-tier provenance and its source identity. */ +export function validateAutoroutingProvenance(value: unknown): AutoroutingLocalIssue[] { + const issues: AutoroutingLocalIssue[] = []; + if (!isRecord(value)) { + issues.push(issue("", "config_invalid", "Expected an autorouting provenance object.")); + return issues; + } + for (const key of Object.keys(value)) { + if (key !== "schema" && key !== "source" && key !== "declarationFingerprint" && key !== "tiersFingerprint") { + issues.push(issue(key, "config_invalid", "Unknown autorouting provenance key.")); + } + } + if (value.schema !== 1) issues.push(issue("schema", "config_invalid", "Expected schema version 1.")); + if (!isRecord(value.source)) { + issues.push(issue("source", "config_invalid", "Expected a provenance source object.")); + } else { + for (const key of Object.keys(value.source)) { + if (key !== "catalogFingerprint" && key !== "mapFingerprint" && key !== "generatorVersion") { + issues.push(issue(`source.${key}`, "config_invalid", "Unknown provenance source key.")); + } + } + validateFingerprint("source.catalogFingerprint", value.source.catalogFingerprint, issues); + validateFingerprint("source.mapFingerprint", value.source.mapFingerprint, issues); + if ( + typeof value.source.generatorVersion !== "number" || + !Number.isSafeInteger(value.source.generatorVersion) || + value.source.generatorVersion < 1 + ) { + issues.push(issue("source.generatorVersion", "config_invalid", "Expected an integer generator version >= 1.")); + } + } + validateFingerprint("declarationFingerprint", value.declarationFingerprint, issues); + validateFingerprint("tiersFingerprint", value.tiersFingerprint, issues); + return issues; +} + +/** Validate only local types, keys, and selector grammar for one source layer. */ +export function validateAutoroutingLocal(fragment: unknown): AutoroutingLocalIssue[] { + const issues: AutoroutingLocalIssue[] = []; + if (fragment === undefined) return issues; + if (!isRecord(fragment)) { + issues.push(issue("", "config_invalid", "Expected task.autorouting to be an object.")); + return issues; + } + + for (const key of Object.keys(fragment)) { + if (!new Set(["enabled", "tiers", "setup", "provenance"]).has(key)) { + issues.push(issue(key, "config_invalid", "Unknown autorouting setting key.")); + } + } + if (fragment.enabled !== undefined && typeof fragment.enabled !== "boolean") { + issues.push(issue("enabled", "config_invalid", "Expected a boolean.")); + } + + if (fragment.setup !== undefined) { + for (const setupIssue of validateAutoroutingSetup(fragment.setup)) { + issues.push({ ...setupIssue, path: setupIssue.path ? `setup.${setupIssue.path}` : "setup" }); + } + } + if (fragment.provenance !== undefined) { + for (const provenanceIssue of validateAutoroutingProvenance(fragment.provenance)) { + issues.push({ + ...provenanceIssue, + path: provenanceIssue.path ? `provenance.${provenanceIssue.path}` : "provenance", + }); + } + } + if (fragment.tiers === undefined) return issues; + if (!isRecord(fragment.tiers)) { + issues.push(issue("tiers", "config_invalid", "Expected an object with only fast, balanced, and strong keys.")); + return issues; + } + for (const key of Object.keys(fragment.tiers)) { + if (!AUTOROUTING_TIERS.includes(key as AutoroutingTier)) { + issues.push(issue(`tiers.${key}`, "config_invalid", "Unknown tier key; expected fast, balanced, or strong.")); + continue; + } + validateSelectorValue(`tiers.${key}`, fragment.tiers[key], issues); + } + return issues; +} + +/** Return a canonical JSON representation with sorted object keys and stored array order. */ +function canonicalAutoroutingJson(value: unknown): string { + if (value === null) return "null"; + if (value === undefined) return "undefined"; + switch (typeof value) { + case "string": + case "boolean": + return JSON.stringify(value); + case "number": + return Number.isFinite(value) ? JSON.stringify(value) : "null"; + case "bigint": + throw new TypeError("Cannot canonicalize bigint"); + case "function": + case "symbol": + return "undefined"; + case "object": + if (Array.isArray(value)) { + return `[${value + .map(item => { + const encoded = canonicalAutoroutingJson(item); + return encoded === "undefined" ? "null" : encoded; + }) + .join(",")}]`; + } + return `{${Object.keys(value as Record) + .sort() + .flatMap(key => { + const encoded = canonicalAutoroutingJson((value as Record)[key]); + return encoded === "undefined" ? [] : [`${JSON.stringify(key)}:${encoded}`]; + }) + .join(",")}}`; + default: + return "undefined"; + } +} + +function autoroutingSha256(value: unknown): string { + return createHash("sha256") + .update(new TextEncoder().encode(canonicalAutoroutingJson(value))) + .digest("hex"); +} + +export type AutoroutingProvenanceState = { + staleMap: boolean; + staleCatalog: boolean; + handEdited: boolean; +}; + +/** Compare recorded provenance with the current catalog/map/tier materialization. */ +export function evaluateAutoroutingProvenanceState( + provenance: AutoroutingProvenance | undefined, + current: { catalogFingerprint: string; mapFingerprint: string; tiers: unknown }, +): AutoroutingProvenanceState { + if (!provenance) return { staleMap: false, staleCatalog: false, handEdited: false }; + return { + staleMap: provenance.source.mapFingerprint !== current.mapFingerprint, + staleCatalog: provenance.source.catalogFingerprint !== current.catalogFingerprint, + handEdited: provenance.tiersFingerprint !== autoroutingSha256(current.tiers), + }; +} + +/** Compare a recorded tier fingerprint with the current raw tier map. */ +export function matchesRecordedTiersFingerprint( + provenance: AutoroutingProvenance | undefined, + tiers: unknown, +): boolean { + return provenance !== undefined && provenance.tiersFingerprint === autoroutingSha256(tiers); +} + +/** Advisory comparison between a recorded declaration and the current provider priority. */ +export type AutoroutingProviderOrderHint = { + /** The declaration lists the same providers in a different relative order. */ + reordered: boolean; + /** Declared providers that the current catalog no longer offers, in declaration order. */ + missing: string[]; +}; + +/** + * Compare a recorded `setup.providers` declaration against the current provider + * priority, for an advisory panel hint only. + * + * Pure string comparison over two arrays, normalized exactly the way provider + * selection normalizes ids (trim + lowercase). It deliberately never reaches + * provenance, effective state, routing, preflight, or evidence: a changed priority + * is a new suggestion, not proof that persisted tiers went stale. + */ +export function autoroutingProviderOrderHint( + setupProviders: readonly string[], + currentOrder: readonly string[], +): AutoroutingProviderOrderHint { + const normalize = (value: string): string => value.trim().toLowerCase(); + const current = currentOrder.map(normalize).filter(id => id.length > 0); + const currentSet = new Set(current); + const declared: string[] = []; + const missing: string[] = []; + const seen = new Set(); + for (const raw of setupProviders) { + const id = normalize(raw); + if (!id || seen.has(id)) continue; + seen.add(id); + if (currentSet.has(id)) declared.push(id); + else missing.push(raw); + } + // Only the providers common to both sides can disagree about order, so a + // declaration that is a subset in the same relative order is not reordered. + const expected = current.filter(id => seen.has(id)); + const reordered = declared.length === expected.length && declared.some((id, index) => id !== expected[index]); + return { reordered, missing }; +} + +export type AutoroutingSettingsBatchPatch = + | { path: "task.autorouting.tiers"; op: "set"; value: TierMap } + | { path: "task.autorouting.setup"; op: "set"; value: AutoroutingSetup } + | { path: "task.autorouting.provenance"; op: "set"; value: AutoroutingProvenance } + | { path: "task.autorouting.tiers" | "task.autorouting.setup" | "task.autorouting.provenance"; op: "unset" }; + +/** Build the one atomic settings batch used by autorouting Apply/Refresh or Clear. */ +export function buildAutoroutingSettingsBatch( + input: { tiers: TierMap; setup: AutoroutingSetup; provenance: AutoroutingProvenance } | { clear: true }, +): readonly AutoroutingSettingsBatchPatch[] { + if ("clear" in input) { + return [ + { path: "task.autorouting.tiers", op: "unset" }, + { path: "task.autorouting.setup", op: "unset" }, + { path: "task.autorouting.provenance", op: "unset" }, + ]; + } + return [ + { path: "task.autorouting.tiers", op: "set", value: structuredClone(input.tiers) }, + { path: "task.autorouting.setup", op: "set", value: structuredClone(input.setup) }, + { path: "task.autorouting.provenance", op: "set", value: structuredClone(input.provenance) }, + ]; +} + +/** Convenience aliases for controller intents that all use one atomic batch. */ +export const buildAutoroutingApplyPatches = buildAutoroutingSettingsBatch; +export const buildAutoroutingRefreshPatches = buildAutoroutingSettingsBatch; +export const buildAutoroutingClearPatches = () => buildAutoroutingSettingsBatch({ clear: true }); + +/** Build the separate single-key enabled toggle mutation. */ +export function buildAutoroutingEnabledPatch(enabled: boolean): { + path: "task.autorouting.enabled"; + op: "set"; + value: boolean; +} { + return { path: "task.autorouting.enabled", op: "set", value: enabled }; +} + +/** Validate effective enablement and map cross-field semantics. */ +export function validateAutoroutingEffective(fragment: unknown): AutoroutingEffective { + if (fragment === undefined || !isRecord(fragment)) return { active: false }; + if (fragment.enabled === undefined || fragment.enabled === false) return { active: false }; + if (fragment.enabled !== true) { + return { + active: false, + issue: effectiveIssue( + "config_invalid", + "task.autorouting.enabled must be a boolean true to enable autorouting.", + ), + }; + } + if (isMeaningfulTierMap(fragment.tiers)) { + return { active: true, map: normalizeTierMap(fragment.tiers) }; + } + return { + active: false, + issue: effectiveIssue( + "map_absent", + "Autorouting is enabled but has no usable tiers. Generate them from the /model smart-routing panel.", + ), + }; +} diff --git a/packages/coding-agent/src/config/autorouting-generator.ts b/packages/coding-agent/src/config/autorouting-generator.ts new file mode 100644 index 0000000000..11e128a829 --- /dev/null +++ b/packages/coding-agent/src/config/autorouting-generator.ts @@ -0,0 +1,238 @@ +/** + * Pure autorouting chain materialization. + * + * The generator consumes an explicit setup, a curation map, and a complete + * model-registry snapshot. It never reads credentials, the clock, network + * state, or discovery availability. + */ + +import { createHash } from "node:crypto"; +import type { Api, Model } from "@gajae-code/ai/core"; +import { + AUTOROUTING_TIERS, + type AutoroutingSetup, + type AutoroutingTier, + isValidAutoroutingSelector, + type TierMap, +} from "./autorouting-contract"; +import { + type AutoroutingCuratedTierMap, + CURATED_TIER_LABELS, + CURATED_TIER_MAP, + type CuratedTierLabels, + canonicalJsonBytes, + computeMapFingerprint, + type TierAssignment, + type TierEffort, +} from "./autorouting-tier-map"; +import { formatModelString } from "./model-resolver"; + +export type { AutoroutingSetup } from "./autorouting-contract"; +export { canonicalJsonBytes } from "./autorouting-tier-map"; + +export const AUTOROUTING_GENERATOR_VERSION = 1; +export const GENERATOR_VERSION = AUTOROUTING_GENERATOR_VERSION; + +export type AutoroutingGeneratorMap = + | AutoroutingCuratedTierMap + | CuratedTierLabels + | { + labels: CuratedTierLabels; + skips?: Record<`${string}/${string}`, { rationale: string; baseline?: true }>; + skipList?: Record<`${string}/${string}`, { rationale: string; baseline?: true }>; + version: number; + }; +export type AutoroutingCuratedMap = AutoroutingGeneratorMap; + +export type AutoroutingSourceIdentity = { + catalogFingerprint: string; + mapFingerprint: string; + generatorVersion: number; +}; + +export type GeneratedTierChains = { + tiers: TierMap; + declarationFingerprint: string; + tiersFingerprint: string; + sourceIdentity: AutoroutingSourceIdentity; +}; + +type CatalogPair = { provider: string; id: string }; +type Candidate = { + selector: string; + key: string; + rank: number; + providerIndex: number; +}; + +function canonicalJsonHash(value: unknown): string { + return createHash("sha256").update(canonicalJsonBytes(value)).digest("hex"); +} + +function assertSetup(setup: AutoroutingSetup): void { + if (setup === null || typeof setup !== "object" || setup.schema !== 1) { + throw new TypeError("Autorouting setup schema must be 1."); + } + if (!Array.isArray(setup.providers) || setup.providers.length === 0) { + throw new TypeError("Autorouting setup providers must be a non-empty array."); + } + for (const provider of setup.providers) { + if (typeof provider !== "string" || provider.trim() !== provider || provider.length === 0) { + throw new TypeError("Autorouting setup providers must contain non-empty strings."); + } + } + if (setup.models !== undefined) { + if (!Array.isArray(setup.models)) throw new TypeError("Autorouting setup models must be an array when present."); + for (const selector of setup.models) { + if (typeof selector !== "string" || selector.trim() !== selector || selector.length === 0) { + throw new TypeError("Autorouting setup models must contain non-empty strings."); + } + } + } +} + +function mapParts(input: AutoroutingGeneratorMap): AutoroutingCuratedTierMap { + if ("labels" in input && input.labels !== undefined) { + return { labels: input.labels, skips: input.skips ?? input.skipList ?? {}, version: input.version }; + } + if (input === CURATED_TIER_LABELS) + return { labels: input, skips: CURATED_TIER_MAP.skips ?? {}, version: CURATED_TIER_MAP.version }; + return { labels: input, skips: {}, version: 1 }; +} + +function lowercaseKey(value: string): string { + return value.toLowerCase(); +} + +function catalogKey(model: Pick, "provider" | "id">): string { + return lowercaseKey(`${model.provider}/${model.id}`); +} + +function compareLex(left: string, right: string): number { + return left === right ? 0 : left < right ? -1 : 1; +} + +function compareCatalogPairs(left: CatalogPair, right: CatalogPair): number { + const leftKey = `${left.provider}/${left.id}`; + const rightKey = `${right.provider}/${right.id}`; + return compareLex(leftKey, rightKey); +} + +function catalogFingerprint(catalog: readonly Model[]): string { + const pairs = catalog.map(model => ({ provider: model.provider, id: model.id })).sort(compareCatalogPairs); + return canonicalJsonHash(pairs); +} + +function splitAllowlistSelector(selector: string): string { + const suffixMatch = selector.match(/:(minimal|low|medium|high|xhigh)$/u); + return suffixMatch ? selector.slice(0, -suffixMatch[0].length) : selector; +} + +function buildAllowlist(setup: AutoroutingSetup): Set | undefined { + if (setup.models === undefined) return undefined; + return new Set(setup.models.map(selector => lowercaseKey(splitAllowlistSelector(selector)))); +} + +function selectorWithEffort(model: Model, effort: TierEffort | undefined): string { + const selector = formatModelString(model); + const generated = effort === undefined ? selector : `${selector}:${effort}`; + if (!isValidAutoroutingSelector(generated)) { + throw new Error(`Generated selector does not match autorouting grammar or length bound: ${generated}`); + } + return generated; +} + +function providerOrder(setup: AutoroutingSetup): readonly string[] { + const seen = new Set(); + const ordered: string[] = []; + for (const provider of setup.providers) { + const normalized = lowercaseKey(provider); + if (seen.has(normalized)) continue; + seen.add(normalized); + ordered.push(provider); + } + return ordered; +} + +function assignmentsForProvider( + labels: CuratedTierLabels, + provider: string, + providerIndex: number, + catalogByKey: ReadonlyMap>, + allowlist: Set | undefined, +): Map { + const byTier = new Map(); + const providerPrefix = lowercaseKey(provider); + for (const [key, assignments] of Object.entries(labels)) { + if (!lowercaseKey(key).startsWith(`${providerPrefix}/`)) continue; + const model = catalogByKey.get(lowercaseKey(key)); + if (!model) continue; + if (allowlist !== undefined && !allowlist.has(lowercaseKey(key))) continue; + for (const assignment of assignments as readonly TierAssignment[]) { + const selector = selectorWithEffort(model, assignment.effort); + const candidates = byTier.get(assignment.tier) ?? []; + candidates.push({ selector, key, rank: assignment.rank, providerIndex }); + byTier.set(assignment.tier, candidates); + } + } + return byTier; +} + +function materializeTiers(setup: AutoroutingSetup, labels: CuratedTierLabels, catalog: readonly Model[]): TierMap { + const catalogByKey = new Map>(); + for (const model of catalog) catalogByKey.set(catalogKey(model), model); + const allowlist = buildAllowlist(setup); + const tierCandidates = new Map(); + for (const [providerIndex, provider] of providerOrder(setup).entries()) { + const providerTiers = assignmentsForProvider(labels, provider, providerIndex, catalogByKey, allowlist); + for (const tier of AUTOROUTING_TIERS) { + const candidates = providerTiers.get(tier); + if (!candidates) continue; + const existing = tierCandidates.get(tier) ?? []; + existing.push(...candidates); + tierCandidates.set(tier, existing); + } + } + + const tiers: TierMap = {}; + for (const tier of AUTOROUTING_TIERS) { + const candidates = tierCandidates.get(tier) ?? []; + candidates.sort( + (left, right) => + left.providerIndex - right.providerIndex || + left.rank - right.rank || + compareLex(lowercaseKey(left.key), lowercaseKey(right.key)) || + compareLex(left.key, right.key), + ); + const selectors: string[] = []; + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate.selector)) continue; + seen.add(candidate.selector); + selectors.push(candidate.selector); + } + if (selectors.length > 0) tiers[tier] = selectors; + } + return tiers; +} + +/** Generate deterministic fast/balanced/strong fallback chains. */ +export function generateTierChains( + setup: AutoroutingSetup, + curatedMap: AutoroutingGeneratorMap = CURATED_TIER_MAP, + catalog: readonly Model[], +): GeneratedTierChains { + assertSetup(setup); + const map = mapParts(curatedMap); + const tiers = materializeTiers(setup, map.labels, catalog); + return { + tiers, + declarationFingerprint: canonicalJsonHash(setup), + tiersFingerprint: canonicalJsonHash(tiers), + sourceIdentity: { + catalogFingerprint: catalogFingerprint(catalog), + mapFingerprint: computeMapFingerprint(map), + generatorVersion: AUTOROUTING_GENERATOR_VERSION, + }, + }; +} diff --git a/packages/coding-agent/src/config/autorouting-tier-map.ts b/packages/coding-agent/src/config/autorouting-tier-map.ts new file mode 100644 index 0000000000..e08d05329b --- /dev/null +++ b/packages/coding-agent/src/config/autorouting-tier-map.ts @@ -0,0 +1,6108 @@ +/** + * Hand-curated autorouting tier labels and the CI baseline skip list. + * + * This file is intentionally data-heavy: labels are reviewed curation, while + * baseline skips are a snapshot of every currently uncurated catalog key. + * Keep generation and validation deterministic; no runtime credentials or + * discovery state belong here. + */ + +import { createHash } from "node:crypto"; +import { getBundledModels, getBundledProviders, type Model } from "@gajae-code/ai/core"; +import { + AUTOROUTING_SELECTOR_PATTERN, + AUTOROUTING_TIERS, + type AutoroutingTier, + isValidAutoroutingSelector, +} from "./autorouting-contract"; + +export type TierMapKey = `${string}/${string}`; +export type TierEffort = "minimal" | "low" | "medium" | "high" | "xhigh"; +export type TierAssignment = { + tier: AutoroutingTier; + effort?: TierEffort; + rank: number; +}; +export type TierMapSkip = { rationale: string; baseline?: true }; + +export type CuratedTierLabels = Record; +export type TierMapSkipList = Record; + +export type AutoroutingCuratedTierMap = { + labels: CuratedTierLabels; + skips?: TierMapSkipList; + skipList?: TierMapSkipList; + version: number; +}; + +type NormalizedAutoroutingCuratedTierMap = { + labels: CuratedTierLabels; + skips: TierMapSkipList; + version: number; +}; + +export const TIER_MAP_VERSION = 1; + +export const CURATED_TIER_LABELS = { + "anthropic/claude-haiku-4-5": [{ tier: "fast", rank: 1 }], + "anthropic/claude-sonnet-5": [{ tier: "balanced", rank: 1 }], + "anthropic/claude-sonnet-4-6": [{ tier: "balanced", rank: 2 }], + "anthropic/claude-opus-5": [{ tier: "strong", effort: "high", rank: 1 }], + "anthropic/claude-opus-4-8": [{ tier: "strong", effort: "high", rank: 2 }], + "openai-codex/gpt-5.6-terra": [ + { tier: "fast", effort: "low", rank: 1 }, + { tier: "balanced", effort: "medium", rank: 1 }, + ], + "openai-codex/gpt-5.6-sol": [{ tier: "strong", effort: "high", rank: 1 }], + "google/gemini-3.5-flash-lite": [{ tier: "fast", rank: 1 }], + "google/gemini-2.5-flash-lite": [{ tier: "fast", rank: 2 }], + "google/gemini-3.5-flash": [{ tier: "balanced", rank: 1 }], + "google/gemini-2.5-flash": [{ tier: "balanced", rank: 2 }], + "google/gemini-3.1-pro-preview": [{ tier: "strong", rank: 1 }], + "google/gemini-2.5-pro": [{ tier: "strong", rank: 2 }], + "xai/grok-4.5": [ + { tier: "fast", effort: "low", rank: 1 }, + { tier: "balanced", effort: "medium", rank: 1 }, + { tier: "strong", effort: "high", rank: 1 }, + ], + "xai/grok-4.3": [ + { tier: "fast", effort: "low", rank: 2 }, + { tier: "balanced", effort: "medium", rank: 2 }, + { tier: "strong", effort: "high", rank: 2 }, + ], +} as const satisfies CuratedTierLabels; + +/** Generated from packages/ai/src/models.json at feature land: 3919 in-scope baseline skips. */ +export const TIER_MAP_SKIP_LIST = { + "cloudflare-ai-gateway/openai/gpt-4.1": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-4.1-mini": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-4.1-nano": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5-mini": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5-nano": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5-pro": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.2-chat-latest": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/openai/gpt-5.2-pro": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.3-chat-latest": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/openai/gpt-5.3-codex-spark": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/openai/gpt-5.4-mini": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.4-nano": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.4-pro": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.5-pro": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.6": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/openai/o1-pro": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/workers-ai/@cf/google/gemma-4-26b-a4b-it": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/ibm-granite/granite-4.0-h-micro": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/meta/llama-3.3-70b-instruct-fp8-fast": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/meta/llama-4-scout-17b-16e-instruct": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/mistralai/mistral-small-3.1-24b-instruct": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/moonshotai/kimi-k2.7-code": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/openai/gpt-oss-120b": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/openai/gpt-oss-20b": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/qwen/qwen3-30b-a3b-fp8": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "deepinfra/Qwen/Qwen3-30B-A3B": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/deepseek-ai/DeepSeek-V3-0324": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "github-copilot/gemini-3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google/gemini-3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/google/gemini-3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-go/glm-5.3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-zen/gemini-3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-zen/muse-spark-1.2": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/google/gemini-3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/google/gemini-3.7-flash:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/nvidia/nemotron-3.5-lightning": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "venice/deepseek-v4-pro-0813": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "venice/gemini-3-7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "venice/nvidia-nemotron-3-5-lightning-30b-a3b": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "venice/qwen-3-8-2-4t-a95b": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3.8-2.4t-a95b": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "vercel-ai-gateway/google/gemini-3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "zai/glm-5.3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gemini-3.7-flash-high": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cursor/gemini-3.7-flash-low": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cursor/gemini-3.7-flash-medium": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google-antigravity/gemini-3.7-flash-high": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google-antigravity/gemini-3.7-flash-low": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google-antigravity/gemini-3.7-flash-medium": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google-antigravity/gemini-3.7-flash-tiered": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google-gemini-cli/gemini-3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/MiniMax-M2.5": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/deepseek-v3.2": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/deepseek-v4-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/glm-5": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/glm-5.1": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/kimi-k2.5": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/kimi-k2.6": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/kimi-k2.7-code": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/qwen3.6-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/qwen3.6-plus": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/qwen3.7-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/qwen3.7-plus": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cloudflare-ai-gateway/anthropic/claude-opus-5": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "cursor/claude-opus-5-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-high-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-low-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-medium-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-high-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-low-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-max-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-medium-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-xhigh": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-opus-5-thinking-xhigh-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-thinking-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-thinking-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-thinking-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-thinking-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-thinking-xhigh": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/claude-sonnet-5-xhigh": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.5-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.5-high-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.5-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.5-low-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.5-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.5-medium-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.6-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.6-high-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.6-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.6-low-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.6-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.6-medium-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.6-xhigh": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/cursor-grok-4.6-xhigh-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gemini-3.6-flash-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gemini-3.6-flash-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gemini-3.6-flash-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gemini-3.6-flash-minimal": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/glm-5.2-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/glm-5.2-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-high-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-low-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-max-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-medium-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-none": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-none-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-xhigh": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-luna-xhigh-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-high-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-low-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-max-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-medium-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-none": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-none-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-xhigh": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-sol-xhigh-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-high-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-low-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-max-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-medium": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-medium-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-none": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-none-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-xhigh": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/gpt-5.6-terra-xhigh-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/kimi-k2.7-code": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/kimi-k3-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/kimi-k3-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "cursor/kimi-k3-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/ByteDance/Seed-2.0-code": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/Qwen/Qwen3-235B-A22B-Instruct-2507": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "deepinfra/Qwen/Qwen3.8-Max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/deepseek-ai/DeepSeek-V3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/deepseek-ai/DeepSeek-V3.1": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "deepinfra/google/gemma-4-E4B-it": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/moonshotai/Kimi-K3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/stepfun-ai/Step-3.7-Flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/tencent/Hy3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/thinkingmachines/Inkling": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "deepinfra/thinkingmachines/Inkling-Small": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "github-copilot/gemini-3.6-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "github-copilot/grok-4.5": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "github-copilot/kimi-k3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "github-copilot/mai-code-1.1-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/gemini-2.5-flash-lite": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/gemini-3-flash-agent": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/gemini-3.1-flash-image": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/gemini-3.1-flash-lite": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/gemini-3.5-flash-extra-low": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google-antigravity/gemini-3.5-flash-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/gemini-3.6-flash-high": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/gemini-3.6-flash-low": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/gemini-3.6-flash-medium": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google-antigravity/gemini-3.6-flash-tiered": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google-antigravity/gemini-pro-agent": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/tab_flash_lite_preview": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google-antigravity/tab_jump_flash_lite_preview": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google/deep-research-max-preview-04-2026": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google/deep-research-preview-04-2026": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google/gemini-2.5-computer-use-preview-10-2025": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "google/gemini-3.1-flash-lite-image": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google/gemini-3.1-flash-live-preview": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "google/gemini-robotics-er-1.6-preview": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "groq/qwen/qwen3.6-27b": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/bytedance-seed/seed-2-1-turbo": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/bytedance-seed/seed-2.0-code": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/deepseek/deepseek-v4-flash-0731": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/deepseek/deepseek-v4-pro-0813": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/inclusionai/ling-3.0-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/inclusionai/ling-3.0-tiny:free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/liquid/lfm-2.5-2.6b:free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/meta/muse-glimmer-30b": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/meta/muse-spark-1.2": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/nvidia/nemotron-3.5-lightning": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/nvidia/nemotron-3.5-lightning:free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/qwen/qwen3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/qwen/qwen3.8-2.4t-a95b": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/qwen/qwen3.8-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/sakana/fugu-ultra": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/sakana/sakana-namazu": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/tencent/hy3:free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/thinkingmachines/inkling-small": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/upstage/solar-pro4": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/x-ai/grok-4.6": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "kilo/~deepseek/deepseek-v4-flash-latest": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "mistral/voxtral-small-latest": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "nvidia/abacusai/dracarys-llama-3.1-70b-instruct": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "nvidia/google/gemma-3-4b-it": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "nvidia/mistralai/mistral-7b-instruct-v0.3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "nvidia/nvidia/cosmos-reason2-8b": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "nvidia/nvidia/llama-3.1-nemotron-nano-8b-v1": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "nvidia/nvidia/llama-3.1-nemotron-nano-vl-8b-v1": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "nvidia/nvidia/llama-3.3-nemotron-super-49b-v1": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "nvidia/nvidia/llama-3.3-nemotron-super-49b-v1.5": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "nvidia/nvidia/nemotron-3.5-lightning-30b-a3b": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "nvidia/nvidia/nemotron-nano-12b-v2-vl": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "nvidia/poolside/laguna-xs-2.1": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "nvidia/thinkingmachines/inkling": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "nvidia/upstage/solar-10.7b-instruct": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openai-codex/gpt-daybreak-blue-latest": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-go/gpt-5.6-luna": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-go/hy3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-go/qwen3.8-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-zen/grok-4.6": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-zen/hy3-free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-zen/kimi-k3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-zen/ling-3.0-tiny-free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "opencode-zen/nemotron-3.5-lightning-free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/anthropic/claude-fable-5:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/anthropic/claude-haiku-4.5:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/anthropic/claude-opus-4.1:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/anthropic/claude-opus-4.5:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/anthropic/claude-opus-4.6:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/anthropic/claude-opus-4.7:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/anthropic/claude-opus-4.8:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/anthropic/claude-opus-5:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/anthropic/claude-sonnet-4.5:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/anthropic/claude-sonnet-4.6:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/anthropic/claude-sonnet-5:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/bytedance-seed/seed-2-1-turbo": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/bytedance-seed/seed-2.0-code": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/deepseek/deepseek-v4-flash-0731": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/deepseek/deepseek-v4-pro-0813": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/google/gemini-2.5-flash-lite:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/google/gemini-2.5-flash:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/google/gemini-2.5-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/google/gemini-3-flash-preview:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/google/gemini-3.1-flash-lite:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/google/gemini-3.1-pro-preview:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/google/gemini-3.5-flash-lite:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/google/gemini-3.5-flash:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/google/gemini-3.6-flash:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/inclusionai/ling-3.0-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/inclusionai/ling-3.0-tiny:free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/liquid/lfm-2.5-2.6b:free": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/meta/muse-glimmer-30b": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/minimax/minimax-m3:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/moonshotai/kimi-k2.7-code:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/nvidia/nemotron-3-ultra-550b-a55b:batch": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/nvidia/nemotron-3.5-lightning:free": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "openrouter/openai/gpt-3.5-turbo:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-4-turbo:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-4.1-mini:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-4.1-nano:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-4.1:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-4o-mini:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-4o:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5-codex:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5-mini:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5-nano:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.1:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.2-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.2:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.4-mini:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.4-nano:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.4-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.4:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.5-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.5:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.6-luna-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.6-luna:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.6-sol-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.6-sol:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.6-terra-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5.6-terra:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/gpt-5:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/o1:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/o3-mini-high:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/o3-mini:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/o3-pro:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/o3:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/o4-mini-high:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/openai/o4-mini:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/qwen/qwen3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/qwen/qwen3.8-2.4t-a95b": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/qwen/qwen3.8-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/sakana/sakana-namazu": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/thinkingmachines/inkling-small": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/thinkingmachines/inkling:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/upstage/solar-pro4": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/x-ai/grok-4.6": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/z-ai/glm-5.2:batch": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/~deepseek/deepseek-v4-flash-latest": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "venice/deepseek-v4-flash-0731": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "venice/deepseek-v4-flash-0731-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "venice/grok-4-6": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "venice/kimi-k3-fast-api": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "venice/qwen-3-8-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "venice/seed-2-1-turbo": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3.7-flash": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3.8-max": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/deepseek/deepseek-v4-flash-0731": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "vercel-ai-gateway/deepseek/deepseek-v4-pro-0813": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "vercel-ai-gateway/inclusionai/ling-3.0-flash": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "vercel-ai-gateway/inclusionai/ling-3.0-tiny-free": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "vercel-ai-gateway/meta/muse-glimmer-30b": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/meta/muse-spark-1.2": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/meta/muse-spark-1.2-contributor": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "vercel-ai-gateway/moonshotai/kimi-k3-fast": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/sakana/namazu": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/tencent/hy3": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "vercel-ai-gateway/thinkingmachines/inkling-small": { + rationale: "post-rebase catalog addition from dev; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.6": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "xai/grok-4.6": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "zai/glm-5.2-highspeed": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "alibaba-token-plan/deepseek-v4-flash-0731": { + baseline: true, + rationale: "post-feature catalog addition; not yet curated", + }, + "alibaba-token-plan/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "alibaba-token-plan/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "alibaba-token-plan/qwen3.8-max": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "alibaba-token-plan/qwen3.8-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/anthropic.claude-3-5-haiku-20241022-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/anthropic.claude-3-haiku-20240307-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/anthropic.claude-3-opus-20240229-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/anthropic.claude-3-sonnet-20240229-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/anthropic.claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/anthropic.claude-opus-4-6-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/anthropic.claude-opus-4-7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/anthropic.claude-opus-4-8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/anthropic.claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/anthropic.claude-sonnet-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/au.anthropic.claude-haiku-4-5-20251001-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/au.anthropic.claude-opus-4-6-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/au.anthropic.claude-opus-4-8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/au.anthropic.claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/au.anthropic.claude-sonnet-4-5-20250929-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/au.anthropic.claude-sonnet-4-6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/au.anthropic.claude-sonnet-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/cohere.command-r-plus-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/cohere.command-r-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/deepseek.v3-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/deepseek.v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/deepseek.v3.2-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/eu.anthropic.claude-3-5-haiku-20241022-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-3-5-sonnet-20241022-v2:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-3-7-sonnet-20250219-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-3-haiku-20240307-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-3-opus-20240229-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-3-sonnet-20240229-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/eu.anthropic.claude-haiku-4-5-20251001-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-opus-4-1-20250805-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-opus-4-20250514-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-opus-4-5-20251101-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-opus-4-6-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-opus-4-7": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-opus-4-8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/eu.anthropic.claude-sonnet-4-20250514-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-sonnet-4-6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/eu.anthropic.claude-sonnet-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.amazon.nova-2-lite-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-fable-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-haiku-4-5-20251001-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-opus-4-5-20251101-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-opus-4-6-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-opus-4-7": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-opus-4-8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-opus-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-sonnet-4-20250514-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-sonnet-4-5-20250929-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-sonnet-4-6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/global.anthropic.claude-sonnet-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/google.gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/google.gemma-3-4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/jp.anthropic.claude-haiku-4-5-20251001-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/jp.anthropic.claude-opus-4-7": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/jp.anthropic.claude-opus-4-8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/jp.anthropic.claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/jp.anthropic.claude-sonnet-4-6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/jp.anthropic.claude-sonnet-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/meta.llama3-1-405b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/meta.llama3-1-70b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/meta.llama3-1-8b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/minimax.minimax-m2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/minimax.minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/minimax.minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/mistral.devstral-2-123b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/mistral.magistral-small-2509": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/mistral.ministral-3-14b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/mistral.ministral-3-3b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/mistral.ministral-3-8b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/mistral.mistral-large-2402-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/mistral.mistral-large-3-675b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/mistral.pixtral-large-2502-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/mistral.voxtral-mini-3b-2507": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/mistral.voxtral-small-24b-2507": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/moonshot.kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/moonshotai.kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/nvidia.nemotron-nano-12b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/nvidia.nemotron-nano-3-30b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/nvidia.nemotron-nano-9b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/nvidia.nemotron-super-3-120b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/openai.gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-oss-120b-1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-oss-20b-1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/openai.gpt-oss-safeguard-120b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/openai.gpt-oss-safeguard-20b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/qwen.qwen3-235b-a22b-2507-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/qwen.qwen3-32b-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/qwen.qwen3-coder-30b-a3b-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/qwen.qwen3-coder-480b-a35b-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/qwen.qwen3-coder-next": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/qwen.qwen3-next-80b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/qwen.qwen3-vl-235b-a22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/us.amazon.nova-lite-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/us.amazon.nova-micro-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/us.amazon.nova-premier-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/us.amazon.nova-pro-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-opus-4-1-20250805-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-opus-4-20250514-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-opus-4-6-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-opus-4-7": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-opus-4-8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-sonnet-4-6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.anthropic.claude-sonnet-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.deepseek.r1-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/us.meta.llama3-2-11b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.meta.llama3-2-1b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.meta.llama3-2-3b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.meta.llama3-2-90b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.meta.llama3-3-70b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.meta.llama4-maverick-17b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/us.meta.llama4-scout-17b-instruct-v1:0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "amazon-bedrock/writer.palmyra-x4-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/writer.palmyra-x5-v1:0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/xai.grok-4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/zai.glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/zai.glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "amazon-bedrock/zai.glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-3-5-sonnet-20240620": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-3-5-sonnet-20241022": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-3-haiku-20240307": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-haiku-4-5-20251001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-opus-4-0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-opus-4-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-opus-4-1-20250805": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-opus-4-20250514": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-opus-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-opus-4-5-20251101": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-opus-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-opus-4-7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-sonnet-4-0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-sonnet-4-20250514": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-sonnet-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "anthropic/claude-sonnet-4-5-20250929": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "azure-openai/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "azure-openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "azure-openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "azure-openai/o3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "azure-openai/o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "bizrouter/anthropic/claude-sonnet-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "bizrouter/google/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "bizrouter/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cerebras/gemma-4-31b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cerebras/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cerebras/llama3.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cerebras/qwen-3-235b-a22b-instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cerebras/qwen-3-coder-480b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cerebras/zai-glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cerebras/zai-glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/anthropic/claude-3-5-haiku": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-3-haiku": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-3-opus": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-3-sonnet": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-3.5-haiku": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-3.5-sonnet": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-fable-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-haiku-4-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-opus-4": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-opus-4-1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-opus-4-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-opus-4-6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-opus-4-7": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-opus-4-8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-sonnet-4": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-sonnet-4-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-sonnet-4-6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/anthropic/claude-sonnet-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/claude-sonnet-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/moonshotai/kimi-k3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/o3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/o3-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/openai/o4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cloudflare-ai-gateway/workers-ai/@cf/moonshotai/kimi-k2.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/moonshotai/kimi-k2.6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/nvidia/nemotron-3-120b-a12b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/zai-org/glm-4.7-flash": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cloudflare-ai-gateway/workers-ai/@cf/zai-org/glm-5.2": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cursor/claude-4-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4-sonnet-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.5-opus-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.5-opus-high-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.5-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.5-sonnet-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.6-opus-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.6-opus-high-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.6-opus-high-thinking-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.6-opus-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.6-opus-max-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.6-opus-max-thinking-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.6-sonnet-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-4.6-sonnet-medium-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-thinking-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-thinking-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-thinking-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-thinking-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-thinking-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-fable-5-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-max-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-medium-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-max-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-medium-fast": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cursor/claude-opus-4-7-thinking-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-thinking-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-7-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-max-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-medium-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-max-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-medium-fast": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "cursor/claude-opus-4-8-thinking-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-thinking-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/claude-opus-4-8-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/composer-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/composer-1.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/composer-2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/composer-2.5-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/default": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gemini-3-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gemini-3-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gemini-3.1-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max-medium-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-max-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-codex-mini-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.1-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-codex-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-codex-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-codex-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-codex-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-codex-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-codex-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-codex-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.2-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex-spark-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.3-codex-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-medium-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-mini-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-mini-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-mini-none": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-mini-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-nano-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-nano-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-nano-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-nano-none": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-nano-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-xhigh": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.4-xhigh-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-extra-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-extra-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-high-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-low-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-medium-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-none": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/gpt-5.5-none-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/grok-4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/grok-build-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "cursor/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/deepseek-ai/DeepSeek-R1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/deepseek-ai/DeepSeek-V3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/deepseek-ai/DeepSeek-V4-Flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/deepseek-ai/DeepSeek-V4-Pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/google/gemma-4-26B-A4B-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/google/gemma-4-31B-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "deepinfra/meta-llama/Llama-4-Scout-17B-16E-Instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "deepinfra/MiniMaxAI/MiniMax-M2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/MiniMaxAI/MiniMax-M2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/MiniMaxAI/MiniMax-M3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/moonshotai/Kimi-K2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/moonshotai/Kimi-K2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/moonshotai/Kimi-K2.7-Code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/nvidia/Llama-3.3-Nemotron-Super-49B-v1.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "deepinfra/nvidia/Nemotron-3-Nano-30B-A3B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "deepinfra/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/openai/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3-32B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "deepinfra/Qwen/Qwen3-Max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3-Next-80B-A3B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3.5-122B-A10B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3.5-27B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3.5-35B-A3B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3.5-397B-A17B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3.5-9B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3.6-27B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3.6-35B-A3B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/Qwen/Qwen3.7-Max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/XiaomiMiMo/MiMo-V2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/XiaomiMiMo/MiMo-V2.5-Pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/zai-org/GLM-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/zai-org/GLM-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/zai-org/GLM-4.7-Flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/zai-org/GLM-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/zai-org/GLM-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepinfra/zai-org/GLM-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepseek/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "deepseek/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "firepass/kimi-k2.6-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fireworks/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fireworks/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fireworks/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fireworks/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fireworks/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fireworks/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fireworks/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fugu/fugu": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "fugu/fugu-ultra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-haiku-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-opus-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-opus-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-opus-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-opus-4.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-sonnet-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-sonnet-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-sonnet-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/claude-sonnet-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gemini-3.1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/kimi-k2.7-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "github-copilot/mai-code-1-flash-picker": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/claude-haiku-4-5-20251001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/claude-opus-4-5-20251101": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/claude-sonnet-4-5-20250929": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-gpt-5-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-gpt-5-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-gpt-5-2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-haiku-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-opus-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-opus-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-sonnet-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/duo-chat-sonnet-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/gpt-5-mini-2025-08-07": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "gitlab-duo/gpt-5.1-2025-11-13": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "glm-zcode/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/claude-opus-4-5-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google-antigravity/claude-opus-4-6-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google-antigravity/claude-sonnet-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/claude-sonnet-4-5-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google-antigravity/claude-sonnet-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/claude-sonnet-4-6-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google-antigravity/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/gemini-2.5-flash-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google-antigravity/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/gemini-3-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/gemini-3-pro-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/gemini-3-pro-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/gemini-3.1-pro-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-antigravity/gpt-oss-120b-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-gemini-cli/gemini-2.0-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-gemini-cli/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-gemini-cli/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-gemini-cli/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-gemini-cli/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-gemini-cli/gemini-3.1-flash-lite-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google-gemini-cli/gemini-3.1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-gemini-cli/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-1.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-1.5-flash-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-1.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-2.0-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-2.0-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-2.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-2.5-flash-lite-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google-vertex/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-3.1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google-vertex/gemini-3.1-pro-preview-customtools": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google/gemini-1.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-1.5-flash-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-1.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-2.0-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-2.0-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-2.5-flash-lite-preview-06-17": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-2.5-flash-lite-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google/gemini-2.5-flash-preview-04-17": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-2.5-flash-preview-05-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-2.5-flash-preview-09-2025": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-2.5-pro-preview-05-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-2.5-pro-preview-06-05": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-3.1-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-3.1-flash-lite-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-3.1-pro-preview-customtools": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-flash-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-flash-lite-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-live-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemini-live-2.5-flash-preview-native-audio": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "google/gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemma-4-26b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemma-4-26b-a4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemma-4-26b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemma-4-31b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "google/gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/deepseek-r1-distill-llama-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/gemma2-9b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/groq/compound": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/groq/compound-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/llama-3.1-8b-instant": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/llama-3.3-70b-versatile": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/llama3-70b-8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/llama3-8b-8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/meta-llama/llama-4-maverick-17b-128e-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "groq/meta-llama/llama-4-scout-17b-16e-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "groq/mistral-saba-24b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/moonshotai/kimi-k2-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/moonshotai/kimi-k2-instruct-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/openai/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/openai/gpt-oss-safeguard-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/qwen-qwq-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "groq/qwen/qwen3-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "huggingface/deepseek-ai/DeepSeek-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "huggingface/deepseek-ai/DeepSeek-V3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "huggingface/meta-llama/Llama-3.3-70B-Instruct-Turbo": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "huggingface/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "jetbrains-junie/claude-fable-5": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/claude-opus-4-6": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/claude-opus-4-7": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/claude-opus-4-8": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/claude-opus-5": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/claude-sonnet-4-6": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/claude-sonnet-5": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/gpt-5-2025-08-07": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/gpt-5.2-2025-12-11": { + baseline: true, + rationale: "post-feature catalog addition; not yet curated", + }, + "jetbrains-junie/gpt-5.3-codex": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/gpt-5.4": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/gpt-5.5": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/gpt-5.6-luna": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/gpt-5.6-sol": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "jetbrains-junie/gpt-5.6-terra": { baseline: true, rationale: "post-feature catalog addition; not yet curated" }, + "kilo/~anthropic/claude-fable-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~anthropic/claude-haiku-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~anthropic/claude-opus-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~anthropic/claude-sonnet-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~google/gemini-flash-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~google/gemini-pro-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~moonshotai/kimi-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~openai/gpt-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~openai/gpt-mini-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/~x-ai/grok-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/ai21/jamba-large-1.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/aion-labs/aion-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/aion-labs/aion-1.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/aion-labs/aion-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/aion-labs/aion-3.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/aion-labs/aion-3.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/aion-labs/aion-rp-llama-3.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/alfredpros/codellama-7b-instruct-solidity": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/alibaba/tongyi-deepresearch-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/allenai/molmo-2-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/allenai/olmo-2-0325-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/allenai/olmo-3-32b-think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/allenai/olmo-3-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/allenai/olmo-3-7b-think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/allenai/olmo-3.1-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/allenai/olmo-3.1-32b-think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/alpindale/goliath-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/amazon/nova-2-lite-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/amazon/nova-lite-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/amazon/nova-micro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/amazon/nova-premier-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/amazon/nova-pro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthracite-org/magnum-v4-72b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-3-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-3.5-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-3.5-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-3.7-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-3.7-sonnet:thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-haiku-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4.6-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4.7-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-4.8-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-opus-5-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-sonnet-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-sonnet-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-sonnet-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/anthropic/claude-sonnet-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/coder-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/maestro-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/spotlight": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/trinity-large-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/trinity-large-preview:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/trinity-large-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/trinity-large-thinking:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/trinity-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/arcee-ai/virtuoso-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/baidu/cobuddy:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/baidu/ernie-4.5-21b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/baidu/ernie-4.5-21b-a3b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/baidu/ernie-4.5-300b-a47b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/baidu/ernie-4.5-vl-28b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/baidu/ernie-4.5-vl-424b-a47b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/baidu/qianfan-ocr-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/baidu/qianfan-ocr-fast:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/bytedance-seed/dola-seed-2.0-pro:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/bytedance-seed/seed-1.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/bytedance-seed/seed-1.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/bytedance-seed/seed-2.0-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/bytedance-seed/seed-2.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/bytedance/ui-tars-1.5-7b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/cognitivecomputations/dolphin-mistral-24b-venice-edition": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/cohere/command-a": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/cohere/command-r-08-2024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/cohere/command-r-plus-08-2024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/cohere/command-r7b-12-2024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/cohere/north-mini-code:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/corethink:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepcogito/cogito-v2.1-671b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-chat-v3-0324": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-chat-v3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-r1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-r1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-r1-distill-llama-70b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/deepseek/deepseek-r1-distill-qwen-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v3.1-terminus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v3.1-terminus:exacto": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/deepseek/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v3.2-exp": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v3.2-speciale": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v4-flash:discounted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v4-flash:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/deepseek/deepseek-v4-pro:discounted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/eleutherai/llemma_7b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/essentialai/rnj-1-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/giga-potato": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/giga-potato-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-2.0-flash-001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-2.0-flash-lite-001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-2.5-flash-image": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-2.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-2.5-flash-lite-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/google/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-2.5-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-2.5-pro-preview-05-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3-pro-image": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3-pro-image-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.1-flash-image": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.1-flash-image-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.1-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.1-flash-lite-image": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.1-flash-lite-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.1-pro-preview-customtools": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/google/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemini-3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemma-2-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemma-2-9b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemma-3-12b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemma-3-4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemma-3n-e4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemma-4-26b-a4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/lyria-3-clip-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/google/lyria-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/gryphe/mythomax-l2-13b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/ibm-granite/granite-4.0-h-micro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/ibm-granite/granite-4.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inception/mercury": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inception/mercury-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inception/mercury-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inclusionai/ling-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inclusionai/ling-2.6-1t:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inclusionai/ling-2.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inclusionai/ling-2.6-flash:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inclusionai/ring-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inclusionai/ring-2.6-1t:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inflection/inflection-3-pi": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/inflection/inflection-3-productivity": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kilo-auto/balanced": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kilo-auto/efficient": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kilo-auto/free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kilo-auto/frontier": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kilo-auto/small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kilo/auto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kilo/auto-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kilo/auto-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kwaipilot/kat-coder-air-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kwaipilot/kat-coder-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kwaipilot/kat-coder-pro-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kwaipilot/kat-coder-pro-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/kwaipilot/kat-coder-pro-v2.5:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/liquid/lfm-2-24b-a2b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/liquid/lfm-2.2-6b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/liquid/lfm2-8b-a1b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mancer/weaver": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meituan/longcat-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meituan/longcat-flash-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3.1-405b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3.1-405b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3.1-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3.1-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3.2-11b-vision-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/meta-llama/llama-3.2-1b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3.2-3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-3.3-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-4-maverick": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-4-scout": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-guard-2-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-guard-3-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-guard-4-12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta-llama/llama-guard-4-12b:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/meta/muse-spark-1.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/microsoft/phi-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/microsoft/phi-4-mini-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/microsoft/wizardlm-2-8x22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-01": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-m1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-m2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-m2-her": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-m2.5:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/minimax/minimax-m3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/codestral-2508": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/devstral-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/devstral-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/devstral-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/ministral-14b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/ministral-3b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/ministral-8b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-7b-instruct-v0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-7b-instruct-v0.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-large-2407": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-large-2411": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-large-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-medium-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-medium-3-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-medium-3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-nemo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-saba": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-small-24b-instruct-2501": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/mistralai/mistral-small-2603": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mistral-small-3.1-24b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/mistralai/mistral-small-3.2-24b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/mistralai/mistral-small-creative": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mixtral-8x22b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/mixtral-8x7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/pixtral-large-2411": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/mistralai/voxtral-small-24b-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2-0905:exacto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2.5:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2.6:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k2.7-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/moonshotai/kimi-k3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/morph-warp-grep-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/morph/morph-v3-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/morph/morph-v3-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/neversleep/llama-3.1-lumimaid-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/neversleep/noromaid-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nex-agi/deepseek-v3.1-nex-n1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nex-agi/nex-n2-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nex-agi/nex-n2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nex-agi/nex-n2-pro:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nousresearch/hermes-2-pro-llama-3-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nousresearch/hermes-3-llama-3.1-405b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nousresearch/hermes-3-llama-3.1-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nousresearch/hermes-4-405b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nousresearch/hermes-4-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nvidia/llama-3.1-nemotron-70b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/nvidia/llama-3.1-nemotron-ultra-253b-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/nvidia/llama-3.3-nemotron-super-49b-v1.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/nvidia/nemotron-3-nano-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/nvidia/nemotron-3-super-120b-a12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nvidia/nemotron-3-super-120b-a12b:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/nvidia/nemotron-3-ultra-550b-a55b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nvidia/nemotron-3-ultra-550b-a55b:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/nvidia/nemotron-3.5-content-safety:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "kilo/nvidia/nemotron-nano-12b-v2-vl": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/nvidia/nemotron-nano-9b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-3.5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-3.5-turbo-0613": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-3.5-turbo-16k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-3.5-turbo-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4-0314": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4-1106-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4-turbo-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4.1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4.1-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o-2024-05-13": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o-2024-08-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o-2024-11-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o-audio-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o-mini-2024-07-18": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o-mini-search-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o-search-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-4o:extended": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5-image": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5-image-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.1-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.2-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.3-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.4-image-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.6-luna-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.6-sol-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-5.6-terra-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-audio": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-audio-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-oss-120b:exacto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/gpt-oss-safeguard-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o1-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o3-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o3-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o3-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o4-mini-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openai/o4-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/opengvlab/internvl3-78b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/auto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/auto-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/bodybuilder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/elephant-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/fusion": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/healer-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/hunter-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/owl-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/openrouter/pareto-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/perceptron/perceptron-mk1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/perplexity/sonar": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/perplexity/sonar-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/perplexity/sonar-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/perplexity/sonar-pro-search": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/perplexity/sonar-reasoning-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/poolside/laguna-m.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/poolside/laguna-m.1:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/poolside/laguna-s-2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/poolside/laguna-s-2.1:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/poolside/laguna-xs-2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/poolside/laguna-xs-2.1:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/poolside/laguna-xs.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/poolside/laguna-xs.2:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/prime-intellect/intellect-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-2.5-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-2.5-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-2.5-coder-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-2.5-vl-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-plus-2025-07-28": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-plus-2025-07-28:thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-vl-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen-vl-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen2.5-coder-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen2.5-vl-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen2.5-vl-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-14b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-235b-a22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-235b-a22b-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-235b-a22b-thinking-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-30b-a3b-instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-30b-a3b-thinking-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-coder-30b-a3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-coder-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-coder-next": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-coder-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-coder:exacto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-max-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-next-80b-a3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-next-80b-a3b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-vl-235b-a22b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-vl-235b-a22b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-vl-30b-a3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-vl-30b-a3b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-vl-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-vl-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3-vl-8b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.5-122b-a10b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.5-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.5-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.5-397b-a17b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.5-9b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.5-flash-02-23": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.5-plus-02-15": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.5-plus-20260420": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.6-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.6-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.6-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.6-plus-preview:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.6-plus:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.7-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwen3.7-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/qwen/qwq-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/raifle/sorcererlm-8x22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/reka/reka-edge": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/rekaai/reka-edge": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/rekaai/reka-flash-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/relace/relace-apply-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/relace/relace-search": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/sao10k/l3-euryale-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/sao10k/l3-lunaris-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/sao10k/l3.1-70b-hanami-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/sao10k/l3.1-euryale-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/sao10k/l3.3-euryale-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stealth/claude-opus-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stealth/claude-opus-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stealth/claude-opus-4.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stealth/claude-sonnet-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stealth/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stealth/qwen3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stepfun/step-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stepfun/step-3.5-flash:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stepfun/step-3.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/stepfun/step-3.7-flash:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/switchpoint/router": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/tencent/hunyuan-a13b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/tencent/hy3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/tencent/hy3-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/tencent/hy3-preview:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/thedrummer/cydonia-24b-v4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/thedrummer/rocinante-12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/thedrummer/skyfall-36b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/thedrummer/unslopnemo-12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/thinkingmachines/inkling": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/tngtech/deepseek-r1t2-chimera": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/undi95/remm-slerp-l2-13b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/upstage/solar-pro-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/writer/palmyra-x5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-3-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-3-mini-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4.1-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4.20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4.20-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4.20-multi-agent": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4.20-multi-agent-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-build-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/x-ai/grok-code-fast-1:optimized:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/xiaomi/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/xiaomi/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/xiaomi/mimo-v2-omni:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/xiaomi/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/xiaomi/mimo-v2-pro:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/xiaomi/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/xiaomi/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4.5-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4.5v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4.6:exacto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kilo/z-ai/glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kimi-code/k3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kimi-code/kimi-for-coding": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kimi-code/kimi-k2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kimi-code/kimi-k2-turbo-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kimi-code/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "kimi-code/kimi-k2.7-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/abacusai/Dracarys-72B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/aion-labs/aion-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/aion-labs/aion-1.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/aion-labs/aion-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/aion-labs/aion-2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/aion-labs/aion-rp-llama-3.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Alibaba-NLP/Tongyi-DeepResearch-30B-A3B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/alibaba/qwen3.6-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/alibaba/qwen3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/allenai/molmo-2-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/allenai/olmo-3-32b-think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/allenai/olmo-3.1-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/allenai/olmo-3.1-32b-think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/amazon/nova-2-lite-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/amazon/nova-lite-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/amazon/nova-micro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/amazon/nova-pro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthracite-org/magnum-v2-72b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthracite-org/magnum-v4-72b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-3.5-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-3.5-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-3.7-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-haiku-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-haiku-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-opus-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-opus-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-opus-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-opus-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-opus-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-opus-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-sonnet-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-sonnet-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-sonnet-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/anthropic/claude-sonnet-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/arcee-ai/trinity-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/arcee-ai/trinity-large-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/arcee-ai/trinity-large-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/arcee-ai/trinity-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/asi1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/auto-model": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/auto-model-basic": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/auto-model-premium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/auto-model-standard": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/azure-gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/azure-gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/azure-gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/azure-o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/azure-o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Baichuan-M2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Baichuan4-Air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Baichuan4-Turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/baidu/ernie-4.5-300b-a47b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/baidu/ernie-4.5-vl-28b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/baidu/ernie-5.0-thinking-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/baidu/ernie-x1.1-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/baseten/Kimi-K2-Instruct-FP4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/brave": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/brave-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/brave-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/bytedance-seed/seed-2.0-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/bytedance/doubao-seed-1.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/bytedance/doubao-seed-2.0-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/bytedance/doubao-seed-2.0-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/bytedance/doubao-seed-2.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/bytedance/doubao-seed-2.0-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/bytedance/doubao-seed-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/chutesai/Mistral-Small-3.2-24B-Instruct-2506": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/claude-3-5-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-5-haiku-20241022": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-5-sonnet-20240620": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-5-sonnet-20241022": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-7-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-7-sonnet-20250219": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-7-sonnet-reasoner": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-7-sonnet-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-7-sonnet-thinking:1024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-7-sonnet-thinking:128000": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-7-sonnet-thinking:32768": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-7-sonnet-thinking:8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-3-haiku-20240307": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-haiku-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-haiku-4-5-20251001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-haiku-4-5-20251001-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-1-20250805": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-1-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-1-thinking:1024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-1-thinking:32000": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-1-thinking:32768": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-1-thinking:8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-20250514": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-5-20251101": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-thinking:1024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-thinking:32000": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-thinking:32768": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-opus-4-thinking:8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-20250514": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-5-20250929": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-5-20250929-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/claude-sonnet-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-thinking:1024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-thinking:32768": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-thinking:64000": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/claude-sonnet-4-thinking:8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/codex-auto-review": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/cognitivecomputations/dolphin-2.9.2-qwen2-72b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/cohere/command-r": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/cohere/command-r-plus-08-2024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/command-a-reasoning-08-2025": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/CrucibleLab/L3.3-70B-Loki-V2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepclaude": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepcogito/cogito-v1-preview-qwen-32B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/deepcogito/cogito-v2.1-671b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-ai/DeepSeek-R1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-ai/DeepSeek-V3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-ai/DeepSeek-V3.1-Terminus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-ai/deepseek-v3.2-exp": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-ai/deepseek-v3.2-exp-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/deepseek-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-chat-cheaper": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-math-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-r1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-r1-sambanova": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-reasoner": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-reasoner-cheaper": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-v3-0324": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-chat-v3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-prover-v2-671b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-r1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-reasoner": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-v3.2-exp": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-v3.2-speciale": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-v4-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-v4-pro-cheaper": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/deepseek/deepseek-v4-pro-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/dmind/dmind-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/dmind/dmind-1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Doctor-Shotgun/MS3.2-24B-Magnum-Diamond": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/doubao-1-5-thinking-pro-250415": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-1.5-pro-256k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-1.5-pro-32k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-seed-1-6-250615": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-seed-1-6-flash-250615": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-seed-1-6-thinking-250615": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-seed-1-8-251215": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-seed-2-0-code-preview-260215": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/doubao-seed-2-0-lite-260215": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-seed-2-0-mini-260215": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-seed-2-0-pro-260215": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/doubao-seed-code-preview-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Envoid/Llama-3.05-Nemotron-Tenyxchat-Storybreaker-70B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Envoid/Llama-3.05-NT-Storybreaker-Ministral-70B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/ernie-4.5-8k-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ernie-4.5-turbo-128k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ernie-4.5-turbo-vl-32k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ernie-5.0-thinking-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ernie-5.0-thinking-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ernie-x1-32k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ernie-x1-32k-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ernie-x1-turbo-32k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ernie-x1.1-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/essentialai/rnj-1-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/EVA-UNIT-01/EVA-LLaMA-3.33-70B-v0.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/EVA-UNIT-01/EVA-LLaMA-3.33-70B-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/EVA-UNIT-01/EVA-Qwen2.5-32B-v0.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/EVA-UNIT-01/EVA-Qwen2.5-72B-v0.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/exa-answer": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/exa-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/exa-research-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/failspy/Meta-Llama-3-70B-Instruct-abliterated-v3.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/fastgpt": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/featherless-ai/Qwerky-72B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/GalrionSoftworks/MN-LooseCannon-12B-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/gemini-2.0-flash-001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.0-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.0-flash-thinking-exp-01-21": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/gemini-2.0-flash-thinking-exp-1219": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.0-pro-exp-02-05": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.0-pro-reasoner": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-flash-lite-preview-06-17": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/gemini-2.5-flash-lite-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/gemini-2.5-flash-lite-preview-09-2025-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/gemini-2.5-flash-nothinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-flash-preview-04-17": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-flash-preview-05-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-flash-preview-09-2025": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-flash-preview-09-2025-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-pro-exp-03-25": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-pro-preview-03-25": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-pro-preview-05-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-2.5-pro-preview-06-05": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-3-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-3-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-3-pro-preview-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gemini-exp-1206": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Gemma-3-27B-ArliAI-RPMax-v3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Gemma-3-27B-Big-Tiger-v3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Gemma-3-27B-CardProjector-v4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Gemma-3-27B-Glitter": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Gemma-3-27B-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Gemma-3-27B-it-Abliterated": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Gemma-3-27B-Nidum-Uncensored": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4-air-0111": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4-airx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4-long": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4-plus-0111": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.1v-thinking-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.1v-thinking-flashx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.5-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/GLM-4.5-Air-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/GLM-4.5-Air-Derestricted-Iceblink": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/GLM-4.5-Air-Derestricted-Iceblink-ReExtract": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/GLM-4.5-Air-Derestricted-Iceblink-v2": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/GLM-4.5-Air-Derestricted-Iceblink-v2-ReExtract": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/GLM-4.5-Air-Derestricted-Steam": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/GLM-4.5-Air-Derestricted-Steam-ReExtract": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/glm-4.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.5v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/GLM-4.6-Derestricted-v5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.7-flash-heretic": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-4.7-flashx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-z1-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-z1-airx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/glm-zero-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-2.0-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-2.0-flash-lite-001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-2.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-3-flash-preview-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/google/gemini-3-pro-image-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-3.1-flash-lite-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/google/gemini-3.1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-3.1-pro-preview-customtools": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/google/gemini-3.1-pro-preview-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-3.1-pro-preview-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-flash-1.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-flash-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-flash-lite-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemini-pro-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemma-3-12b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemma-4-26b-a4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/google/gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.3-codex-spark": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/gpt-image-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-3-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-3-fast-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-3-mini-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-3-mini-fast-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-4-1-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-4-1-fast-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-4-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-4-fast-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-imagine-image": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/grok-imagine-video": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Gryphe/MythoMax-L2-13b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:deepseek-ai/DeepSeek-R1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:deepseek-ai/DeepSeek-V3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:deepseek-ai/DeepSeek-V3-0324": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:deepseek-ai/DeepSeek-V3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:meta-llama/Llama-3.3-70B-Instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/hf:MiniMaxAI/MiniMax-M2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:MiniMaxAI/MiniMax-M2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:moonshotai/Kimi-K2-Instruct-0905": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/hf:moonshotai/Kimi-K2-Thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:moonshotai/Kimi-K2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:moonshotai/Kimi-K2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:nvidia/Kimi-K2.5-NVFP4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/hf:openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:Qwen/Qwen3-235B-A22B-Thinking-2507": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/hf:Qwen/Qwen3-Coder-480B-A35B-Instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/hf:Qwen/Qwen3.5-397B-A17B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:zai-org/GLM-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:zai-org/GLM-4.7-Flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:zai-org/GLM-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hf:zai-org/GLM-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/holo3-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/huihui-ai/DeepSeek-R1-Distill-Llama-70B-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/huihui-ai/DeepSeek-R1-Distill-Qwen-32B-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/huihui-ai/Llama-3.1-Nemotron-70B-Instruct-HF-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/huihui-ai/Llama-3.3-70B-Instruct-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/huihui-ai/Qwen2.5-32B-Instruct-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/hunyuan-t1-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/hunyuan-turbos-20250226": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ibm-granite/granite-4.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ling-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ling-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ling-2.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ling-flash-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ling-mini-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/llada2.0-flash-cap": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/llada2.1-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ming-flash-omni-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ring-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ring-flash-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inclusionai/ring-mini-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Infermatic/MN-12B-Inferor-v0.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inflatebot/MN-12B-Mag-Mell-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inflection/inflection-3-pi": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/inflection/inflection-3-productivity": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/jamba-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/jamba-large-1.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/jamba-large-1.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/jamba-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/jamba-mini-1.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/jamba-mini-1.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/KAT-Coder-Air-V1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/KAT-Coder-Exp-72B-1010": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/KAT-Coder-Pro-V1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kimi-k2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kimi-k2-instruct-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kimi-thinking-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kuaishou/kat-coder-pro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kuaishou/kat-coder-pro-v1-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kuaishou/kat-coder-pro-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/kwaipilot/kat-coder-pro-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/LatitudeGames/Wayfarer-Large-70B-Llama-3.3": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/learnlm-1.5-pro-experimental": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/liquid/lfm-2-24b-a2b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Anthrobomination": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Argunaut-1-SFT": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-ArliAI-RPMax-v1.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-ArliAI-RPMax-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-ArliAI-RPMax-v3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Aurora-Borealis": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Bigger-Body": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Cirrus-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Cu-Mai-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Damascus-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Dark-Ages-v0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Electra-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Electranova-v1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Fallen-R1-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Fallen-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Forgotten-Abomination-v5.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-Forgotten-Safeword-3.6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-GeneticLemonade-Opus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-GeneticLemonade-Unleashed-v3": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-Ignition-v0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Incandescent-Malevolence": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-Legion-V2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Magnum-v4-SE": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Magnum-v4-SE-Cirrus-x1-SLERP": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-Mhnnn-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-MiraiFanfare": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Mokume-Gane-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-MS-Nevoria": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Nova": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Predatorial-Extasy": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Progenitor-V3.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-RAWMAW": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Sapphira-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Sapphira-0.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-Shakudo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3-70B-StrawberryLemonade-v1.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-Strawberrylemonade-v1.2": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-The-Omega-Directive-Unslop-v2.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-The-Omega-Directive-Unslop-v2.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3-70B-Vulpecula-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3+(3.1v3.3)-70B-Hanami-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Llama-3.3+(3.1v3.3)-70B-New-Dawn-v1.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Llama-3.3+(3v3.3)-70B-TenyxChat-DaybreakStorywriter": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/LLM360/K2-Think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Magistral-Small-2506": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/MarinaraSpaghetti/NemoMix-Unleashed-12B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/meganova-ai/manta-flash-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meganova-ai/manta-mini-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meganova-ai/manta-pro-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meituan-longcat/LongCat-Flash-Chat-FP8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/mercury-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mercury-coder-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Meta-Llama-3-1-8B-Instruct-FP8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meta-llama/llama-3.1-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meta-llama/llama-3.2-3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meta-llama/llama-3.3-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meta-llama/llama-4-maverick": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meta-llama/llama-4-scout": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meta/llama-3.3-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/meta/llama-4-scout-17b-16e-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/microsoft/MAI-DS-R1-FP8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/microsoft/wizardlm-2-8x22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/MiniMax-M1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/MiniMax-M2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-01": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-m2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-m2-her": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-m2.5-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-m2.7-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/minimax/minimax-m2.7-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/MiniMaxAI/MiniMax-M1-80k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/miromind-ai/mirothinker-v1.5-235b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Mistral-Nemo-12B-Instruct-2407": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistral-small-31-24b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistral/mistral-medium-3.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistral/mistral-vibe-cli-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/codestral-2508": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/devstral-2-123b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/mistralai/Devstral-Small-2505": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/ministral-14b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/ministral-14b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/mistralai/ministral-3b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/ministral-8b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/mistral-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/mistral-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/mistral-large-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/mistral-large-3-675b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/mistralai/mistral-medium-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/mistral-medium-3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/Mistral-Nemo-Instruct-2407": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/mistralai/mistral-saba": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/mistral-small-4-119b-2603": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/mistralai/mistral-small-creative": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/mistral-tiny": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/mistralai/mixtral-8x22b-instruct-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/mistralai/mixtral-8x7b-instruct-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/mlabonne/NeuralDaredevil-8B-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/moonshotai/Kimi-Dev-72B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-k2-0711": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-k2-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-k2-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-k2-instruct-0711": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/Kimi-K2-Instruct-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-k2-thinking-original": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/moonshotai/kimi-k2-thinking-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-k2-thinking-turbo-original": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/moonshotai/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/moonshotai/kimi-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/NeverSleep/Llama-3-Lumimaid-70B-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/NeverSleep/Lumimaid-v0.2-70B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/nex-agi/deepseek-v3.1-nex-n1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/nothingiisreal/L3.1-70B-Celeste-V0.1-BF16": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/NousResearch/DeepHermes-3-Mistral-24B-Preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/nousresearch/hermes-3-llama-3.1-70b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/nousresearch/hermes-4-405b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/nousresearch/hermes-4-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/nvidia/Llama-3_3-Nemotron-Super-49B-v1_5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/nvidia/nemotron-3-nano-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/nvidia/nemotron-3-super-120b-a12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/nvidia/nvidia-nemotron-nano-9b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/chatgpt-4o-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-3.5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4-turbo-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4.1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4.1-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4o-2024-08-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4o-2024-11-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4o-mini-search-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-4o-search-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.1-2025-11-13": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.1-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.1-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.2-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.3-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-5.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/gpt-oss-safeguard-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o1-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o1-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o3-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o3-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o3-mini-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o3-pro-2025-06-10": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o4-mini-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/openai/o4-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/owl": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/pamanseau/OpenReasoning-Nemotron-32B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/pangram-ai-detection": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/pangram-plagiarism-detection": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/phi-4-mini-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/phi-4-multimodal-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/poolside/laguna-m.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/poolside/laguna-xs.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qvq-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen-3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen-long": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen-2.5-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen2.5-Coder-32B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-14b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-235b-a22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen3-235B-A22B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-235b-a22b-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen3-235B-A22B-Instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen3-235B-A22B-Instruct-2507-TEE": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/qwen/qwen3-235b-a22b-thinking-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen3-235B-A22B-Thinking-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen3-8B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-coder-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-coder-next": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-coder-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen3-Next-80B-A3B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-next-80b-a3b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen3-VL-235B-A22B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3-vl-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.5-397b-a17b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.5-397b-a17b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.5-9b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.5-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.5-plus-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen/Qwen3.6-35B-A3B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.6-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwen3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen/qwq-32b-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen2.5-32B-EVA-v0.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen25-vl-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3-30b-a3b-instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3-coder-30b-a3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3-max-2026-01-23": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3-vl-235b-a22b-instruct-original": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/qwen3-vl-235b-a22b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3.5-122b-a10b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3.5-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-Anko": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-BlueStar-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-BlueStar-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-BlueStar-v2-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-BlueStar-v2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-BlueStar-v3-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-BlueStar-v3-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-earica-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-earica-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Infracelestial": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-Marvin-DPO-V2-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Marvin-DPO-V2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Marvin-V2-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-Marvin-V2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Musica-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-NaNovel-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-NaNovel-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Omega-Evolution-v2.0-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Omega-Evolution-v2.0-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Queen-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-Queen-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-RpRMax-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-Vivid-Durian": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-Writer-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-Writer-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Qwen3.5-27B-Writer-V2-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Qwen3.5-27B-Writer-V2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/qwen3.5-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3.5-omni-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3.5-omni-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwen3.6-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/qwq-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/QwQ-32B-ArliAI-RpR-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/raifle/sorcererlm-8x22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/ReadyArt/MS3.2-The-Omega-Directive-24B-Unslop-v2.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/ReadyArt/The-Omega-Abomination-L-70B-v1.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/Salesforce/Llama-xLAM-2-70b-fc-r": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Sao10K/L3-8B-Stheno-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Sao10K/L3.1-70B-Euryale-v2.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Sao10K/L3.1-70B-Hanami-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Sao10K/L3.3-70B-Euryale-v2.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/sapiens-ai/agnes-1.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/sapiens-ai/agnes-1.5-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/sapiens-ai/agnes-1.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/sarvan-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/shisa-ai/shisa-v2-llama3.3-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/shisa-ai/shisa-v2.1-llama3.3-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/sonar": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/sonar-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/sonar-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/sonar-reasoning-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/soob3123/amoral-gemma3-27B-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/soob3123/GrayLine-Qwen3-8B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/soob3123/Veiled-Calla-12B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Steelskull/L3.3-Cu-Mai-R1-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Steelskull/L3.3-Electra-R1-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Steelskull/L3.3-MS-Evalebis-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Steelskull/L3.3-MS-Evayale-70B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Steelskull/L3.3-MS-Nevoria-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Steelskull/L3.3-Nevoria-R1-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/step-2-16k-exp": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/step-2-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/step-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/step-r1-v-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/stepfun-ai/step-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/stepfun-ai/step-3.5-flash-2603": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/stepfun/step-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/stepfun/step-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/stepfun/step-3.5-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/study_gpt-chatgpt-4o-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/deepseek-r1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/deepseek-v3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/gemma4-31b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/glm-5-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/glm-5-1-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/glm-5.1-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/kimi-k2.5-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/llama3-3-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/qwen2.5-vl-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/qwen3-30b-a3b-instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/qwen3-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/qwen3-coder-next": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/qwen3.5-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TEE/qwen3.5-397b-a17b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/tencent/hunyuan-2.0-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/tencent/Hunyuan-MT-7B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/tencent/hy3-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Anubis-70B-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Anubis-70B-v1.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Cydonia-24B-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Cydonia-24B-v4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Cydonia-24B-v4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Cydonia-24B-v4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Magidonia-24B-v4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Rocinante-12B-v1.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/Skyfall-31B-v4.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/thedrummer/skyfall-36b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/TheDrummer/UnslopNemo-12B-v4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/THUDM/GLM-4-32B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/THUDM/GLM-4-9B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/THUDM/GLM-Z1-32B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/THUDM/GLM-Z1-9B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/THUDM/GLM-Z1-Rumination-32B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/tngtech/DeepSeek-TNG-R1T2-Chimera": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/tngtech/tng-r1t-chimera": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/Tongyi-Zhiwen/QwenLong-L1-32B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/undi95/remm-slerp-l2-13b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/universal-summarizer": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/unsloth/gemma-3-12b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/unsloth/gemma-3-1b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/unsloth/gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/unsloth/gemma-3-4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/upstage/solar-pro-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/v0-1.0-md": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/v0-1.5-lg": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/v0-1.5-md": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/venice-uncensored": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/venice-uncensored:web": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/volcengine/doubao-seed-1-6-vision": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/volcengine/doubao-seed-1.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/volcengine/doubao-seed-2.0-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/volcengine/doubao-seed-2.0-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/volcengine/doubao-seed-2.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/volcengine/doubao-seed-2.0-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/volcengine/doubao-seed-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/VongolaChouko/Starcannon-Unleashed-12B-v1.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/x-ai/grok-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4-07-09": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.1-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.1-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.1-fast-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.2-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.2-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.20-beta-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.20-beta-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.20-multi-agent": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.20-multi-agent-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/x-ai/grok-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/xiaomi/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/xiaomi/mimo-v2-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/xiaomi/mimo-v2-flash-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/xiaomi/mimo-v2-flash-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/xiaomi/mimo-v2-flash-thinking-original": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "litellm/xiaomi/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/xiaomi/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/xiaomi/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/xiaomi/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/yi-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/yi-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/yi-medium-200k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.5-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.5v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.6v-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.6v-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.7-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-4.7-flashx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/z-ai/glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/GLM-4.5-Air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.6-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/GLM-4.6-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.6v-flash-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.6v-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.7-flash-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-4.7-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-5-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "litellm/zai-org/glm-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mara/DeepSeek-V3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mara/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mara/MiniMax-M2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mara/MiniMax-M2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-cn/MiniMax-M2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-cn/MiniMax-M2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-cn/MiniMax-M2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-cn/MiniMax-M2.5-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-cn/MiniMax-M2.5-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-cn/MiniMax-M2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-cn/MiniMax-M2.7-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-cn/MiniMax-M3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M2.1-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M2.5-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M2.5-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M2.7-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code-cn/MiniMax-M3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M2.1-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M2.5-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M2.5-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M2.7-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax-code/MiniMax-M3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax/MiniMax-M2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax/MiniMax-M2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax/MiniMax-M2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax/MiniMax-M2.5-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax/MiniMax-M2.5-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax/MiniMax-M2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax/MiniMax-M2.7-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "minimax/MiniMax-M3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/codestral-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/devstral-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/devstral-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/devstral-medium-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/devstral-medium-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/devstral-small-2505": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/devstral-small-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/labs-devstral-small-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/magistral-medium-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/magistral-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/ministral-3b-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/ministral-8b-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-large-2411": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-large-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-large-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-medium-2505": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-medium-2508": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-medium-2604": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-medium-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-nemo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-small-2506": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-small-2603": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/mistral-small-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/open-mistral-7b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/open-mistral-nemo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/open-mixtral-8x22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/open-mixtral-8x7b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/pixtral-12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "mistral/pixtral-large-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "moonshot/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/abacusai/Dracarys-72B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/aion-labs/aion-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/aion-labs/aion-1.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/aion-labs/aion-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/aion-labs/aion-2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/aion-labs/aion-rp-llama-3.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Alibaba-NLP/Tongyi-DeepResearch-30B-A3B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/alibaba/qwen3.6-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/alibaba/qwen3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/allenai/molmo-2-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/allenai/olmo-3-32b-think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/allenai/olmo-3.1-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/allenai/olmo-3.1-32b-think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/amazon/nova-2-lite-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/amazon/nova-lite-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/amazon/nova-micro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/amazon/nova-pro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/anthracite-org/magnum-v2-72b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/anthracite-org/magnum-v4-72b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/anthropic/claude-haiku-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/anthropic/claude-opus-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/anthropic/claude-opus-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/anthropic/claude-opus-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/anthropic/claude-sonnet-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/anthropic/claude-sonnet-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/arcee-ai/trinity-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/arcee-ai/trinity-large-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/arcee-ai/trinity-large-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/arcee-ai/trinity-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/asi1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/auto-model": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/auto-model-basic": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/auto-model-premium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/auto-model-standard": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/azure-gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/azure-gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/azure-gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/azure-o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/azure-o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Baichuan-M2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Baichuan4-Air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Baichuan4-Turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/baidu/ernie-4.5-300b-a47b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/baidu/ernie-4.5-vl-28b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/baseten/Kimi-K2-Instruct-FP4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/brave": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/brave-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/brave-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/bytedance-seed/seed-2.0-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/chutesai/Mistral-Small-3.2-24B-Instruct-2506": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/claude-3-5-haiku-20241022": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-5-sonnet-20240620": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-5-sonnet-20241022": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-7-sonnet-20250219": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-7-sonnet-reasoner": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-7-sonnet-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-7-sonnet-thinking:1024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-7-sonnet-thinking:128000": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-7-sonnet-thinking:32768": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-3-7-sonnet-thinking:8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-haiku-4-5-20251001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-haiku-4-5-20251001-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-1-20250805": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-1-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-1-thinking:1024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-1-thinking:32000": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-1-thinking:32768": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-1-thinking:8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-20250514": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-5-20251101": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-thinking:1024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-thinking:32000": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-thinking:32768": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-opus-4-thinking:8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-sonnet-4-20250514": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-sonnet-4-5-20250929": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-sonnet-4-5-20250929-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/claude-sonnet-4-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-sonnet-4-thinking:1024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-sonnet-4-thinking:32768": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-sonnet-4-thinking:64000": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claude-sonnet-4-thinking:8192": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claw-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claw-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/claw-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/cognitivecomputations/dolphin-2.9.2-qwen2-72b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/cohere/command-r": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/cohere/command-r-plus-08-2024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/command-a-plus-05-2026": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/command-a-reasoning-08-2025": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/CrucibleLab/L3.3-70B-Loki-V2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepclaude": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepcogito/cogito-v1-preview-qwen-32B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/deepcogito/cogito-v2.1-671b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-ai/DeepSeek-R1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-ai/DeepSeek-V3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-ai/DeepSeek-V3.1-Terminus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-ai/deepseek-v3.2-exp": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-ai/deepseek-v3.2-exp-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/deepseek-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-chat-cheaper": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-math-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-r1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-r1-sambanova": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-reasoner": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-reasoner-cheaper": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek-v3-0324": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek/deepseek-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek/deepseek-prover-v2-671b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek/deepseek-v3.2-speciale": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/deepseek/deepseek-v4-pro-cheaper": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/dmind/dmind-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/dmind/dmind-1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Doctor-Shotgun/MS3.2-24B-Magnum-Diamond": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/doubao-1-5-thinking-pro-250415": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-1.5-pro-256k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-1.5-pro-32k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-seed-1-6-250615": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-seed-1-6-flash-250615": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-seed-1-6-thinking-250615": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-seed-1-8-251215": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-seed-2-0-code-preview-260215": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/doubao-seed-2-0-lite-260215": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-seed-2-0-mini-260215": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-seed-2-0-pro-260215": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/doubao-seed-code-preview-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Envoid/Llama-3.05-Nemotron-Tenyxchat-Storybreaker-70B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Envoid/Llama-3.05-NT-Storybreaker-Ministral-70B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/ernie-4.5-8k-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-4.5-turbo-128k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-4.5-turbo-vl-32k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-5.0-thinking-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-5.0-thinking-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-x1-32k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-x1-32k-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-x1-turbo-32k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ernie-x1.1-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/essentialai/rnj-1-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/EVA-UNIT-01/EVA-LLaMA-3.33-70B-v0.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/EVA-UNIT-01/EVA-LLaMA-3.33-70B-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/EVA-UNIT-01/EVA-Qwen2.5-32B-v0.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/EVA-UNIT-01/EVA-Qwen2.5-72B-v0.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/exa-answer": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/exa-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/exa-research-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/failspy/Meta-Llama-3-70B-Instruct-abliterated-v3.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/fastgpt": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/featherless-ai/Qwerky-72B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/GalrionSoftworks/MN-LooseCannon-12B-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/gemini-2.0-flash-001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.0-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.0-flash-thinking-exp-01-21": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/gemini-2.0-flash-thinking-exp-1219": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.0-pro-exp-02-05": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.0-pro-reasoner": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-flash-lite-preview-06-17": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/gemini-2.5-flash-lite-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/gemini-2.5-flash-lite-preview-09-2025-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/gemini-2.5-flash-nothinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-flash-preview-04-17": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-flash-preview-05-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-flash-preview-09-2025": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-flash-preview-09-2025-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-pro-exp-03-25": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-pro-preview-03-25": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-pro-preview-05-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-2.5-pro-preview-06-05": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-3-pro-preview-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemini-exp-1206": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-3-27B-ArliAI-RPMax-v3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-3-27B-Big-Tiger-v3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-3-27B-CardProjector-v4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-3-27B-Glitter": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-3-27B-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-3-27B-it-Abliterated": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-3-27B-Nidum-Uncensored": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-4-31B-Claude-4.6-Opus-Reasoning-Distilled": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Gemma-4-31B-Cognitive-Unshackled": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-4-31B-DarkIdol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemma-4-31B-Fabled": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemma-4-31B-Garnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-4-31B-GarnetV2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-4-31B-Gemopus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-4-31B-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemma-4-31B-K1-v5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemma-4-31B-Larkspur-v0.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/gemma-4-31B-MeroMero": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-4-31B-Musica-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gemma-4-31B-Queen": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4-air-0111": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4-airx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4-long": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4-plus-0111": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4.1v-thinking-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4.1v-thinking-flashx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/GLM-4.5-Air-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/GLM-4.5-Air-Derestricted-Iceblink": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/GLM-4.5-Air-Derestricted-Iceblink-ReExtract": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/GLM-4.5-Air-Derestricted-Iceblink-v2": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/GLM-4.5-Air-Derestricted-Iceblink-v2-ReExtract": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/GLM-4.5-Air-Derestricted-Steam": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/GLM-4.5-Air-Derestricted-Steam-ReExtract": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/GLM-4.6-Derestricted-v5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-4.7-flash-heretic": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-z1-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-z1-airx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/glm-zero-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-3-flash-preview-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/google/gemini-3.1-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-3.1-flash-lite-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/google/gemini-3.1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-3.1-pro-preview-customtools": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/google/gemini-3.1-pro-preview-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-3.1-pro-preview-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-3.5-flash-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-flash-1.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-flash-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-flash-lite-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemini-pro-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemma-4-26b-a4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/google/gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/grok-3-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/grok-3-fast-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/grok-3-mini-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/grok-3-mini-fast-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Gryphe/MythoMax-L2-13b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/hermes-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/hermes-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/hermes-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/holo3-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/huihui-ai/DeepSeek-R1-Distill-Llama-70B-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/huihui-ai/DeepSeek-R1-Distill-Qwen-32B-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/huihui-ai/Llama-3.1-Nemotron-70B-Instruct-HF-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/huihui-ai/Llama-3.3-70B-Instruct-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/huihui-ai/Qwen2.5-32B-Instruct-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/hunyuan-t1-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/hunyuan-turbos-20250226": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ibm-granite/granite-4.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/inclusionai/ling-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/inclusionai/ling-2.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/inclusionai/ring-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Infermatic/MN-12B-Inferor-v0.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/inflatebot/MN-12B-Mag-Mell-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/inflection/inflection-3-pi": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/inflection/inflection-3-productivity": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/jamba-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/jamba-large-1.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/jamba-large-1.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/jamba-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/jamba-mini-1.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/jamba-mini-1.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/KAT-Coder-Air-V1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/KAT-Coder-Exp-72B-1010": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/KAT-Coder-Pro-V1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/kimi-k2-instruct-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/kimi-thinking-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/kwaipilot/kat-coder-pro-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/LatitudeGames/Wayfarer-Large-70B-Llama-3.3": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/learnlm-1.5-pro-experimental": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/liquid/lfm-2-24b-a2b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Anthrobomination": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Argunaut-1-SFT": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-ArliAI-RPMax-v1.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-ArliAI-RPMax-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-ArliAI-RPMax-v3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Aurora-Borealis": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Bigger-Body": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Cirrus-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Cu-Mai-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Damascus-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Dark-Ages-v0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Electra-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Electranova-v1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Fallen-R1-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Fallen-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Forgotten-Abomination-v5.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-Forgotten-Safeword-3.6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-GeneticLemonade-Opus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-GeneticLemonade-Unleashed-v3": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-Ignition-v0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Incandescent-Malevolence": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-Legion-V2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Magnum-v4-SE": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Magnum-v4-SE-Cirrus-x1-SLERP": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-Mhnnn-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-MiraiFanfare": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Mokume-Gane-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-MS-Nevoria": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Nova": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Predatorial-Extasy": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Progenitor-V3.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-RAWMAW": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Sapphira-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Sapphira-0.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-Shakudo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3-70B-StrawberryLemonade-v1.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-Strawberrylemonade-v1.2": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-The-Omega-Directive-Unslop-v2.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-The-Omega-Directive-Unslop-v2.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3-70B-Vulpecula-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3+(3.1v3.3)-70B-Hanami-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Llama-3.3+(3.1v3.3)-70B-New-Dawn-v1.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Llama-3.3+(3v3.3)-70B-TenyxChat-DaybreakStorywriter": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/LLM360/K2-Think": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Magistral-Small-2506": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/MarinaraSpaghetti/NemoMix-Unleashed-12B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/meganova-ai/manta-flash-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/meganova-ai/manta-mini-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/meganova-ai/manta-pro-1.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/meituan-longcat/LongCat-Flash-Chat-FP8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/mercury-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mercury-coder-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Meta-Llama-3-1-8B-Instruct-FP8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/meta-llama/llama-3.1-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/meta-llama/llama-3.2-3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/meta-llama/llama-3.3-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/meta-llama/llama-4-maverick": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/meta-llama/llama-4-scout": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/microsoft/MAI-DS-R1-FP8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/microsoft/wizardlm-2-8x22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/MiniMax-M1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/MiniMax-M2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/minimax/minimax-01": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/minimax/minimax-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/minimax/minimax-m2-her": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/minimax/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/minimax/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/minimax/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/minimax/minimax-m2.7-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/MiniMaxAI/MiniMax-M1-80k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/miromind-ai/mirothinker-v1.5-235b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mirothinker-1-7-deepresearch": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mirothinker-1-7-deepresearch-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Mistral-Nemo-12B-Instruct-2407": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistral-small-31-24b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistral/mistral-medium-3.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistral/mistral-vibe-cli-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/codestral-2508": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/devstral-2-123b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/mistralai/Devstral-Small-2505": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/ministral-14b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/ministral-14b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/mistralai/ministral-3b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/ministral-8b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/mistral-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/mistral-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/mistral-large-3-675b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/mistralai/mistral-medium-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/mistral-medium-3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/Mistral-Nemo-Instruct-2407": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/mistralai/mistral-saba": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/mistral-small-4-119b-2603": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/mistralai/mistral-small-creative": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/mistral-tiny": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/mistralai/mixtral-8x22b-instruct-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/mistralai/mixtral-8x7b-instruct-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/mlabonne/NeuralDaredevil-8B-abliterated": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/moonshotai/Kimi-Dev-72B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/moonshotai/kimi-k2-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/moonshotai/kimi-k2-instruct-0711": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/moonshotai/Kimi-K2-Instruct-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/moonshotai/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/moonshotai/kimi-k2-thinking-original": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/moonshotai/kimi-k2-thinking-turbo-original": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/moonshotai/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/moonshotai/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/moonshotai/kimi-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nanogpt/coding-router": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nanogpt/coding-router:high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nanogpt/coding-router:low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nanogpt/coding-router:max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nanogpt/coding-router:medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/NeverSleep/Llama-3-Lumimaid-70B-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/NeverSleep/Lumimaid-v0.2-70B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nex-agi/deepseek-v3.1-nex-n1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nothingiisreal/L3.1-70B-Celeste-V0.1-BF16": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/NousResearch/DeepHermes-3-Mistral-24B-Preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/nousresearch/hermes-3-llama-3.1-70b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/nousresearch/hermes-4-405b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nousresearch/hermes-4-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nvidia/Llama-3_3-Nemotron-Super-49B-v1_5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/nvidia/Llama-3.3-Nemotron-Super-49B-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/nvidia/nemotron-3-nano-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/nvidia/nemotron-3-super-120b-a12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/nvidia/nvidia-nemotron-nano-9b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/chatgpt-4o-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-3.5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4-turbo-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4.1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4.1-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4o-2024-08-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4o-2024-11-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4o-mini-search-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-4o-search-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.1-2025-11-13": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.1-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.1-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.2-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.3-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/gpt-oss-safeguard-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o1-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o1-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o3-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o3-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o3-mini-low": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o3-pro-2025-06-10": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o4-mini-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/openai/o4-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/owl": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/pamanseau/OpenReasoning-Nemotron-32B": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/pangram-ai-detection": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/pangram-plagiarism-detection": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/perceptron/perceptron-mk1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/phi-4-mini-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/phi-4-multimodal-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/poolside/laguna-m.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/poolside/laguna-xs.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qvq-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen-3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen-long": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen-2.5-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen/Qwen2.5-Coder-32B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-14b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-235b-a22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen/Qwen3-235B-A22B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen/Qwen3-235B-A22B-Instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen/Qwen3-235B-A22B-Instruct-2507-TEE": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen/Qwen3-235B-A22B-Thinking-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen/Qwen3-8B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-coder-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-coder-next": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-coder-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen/Qwen3-Next-80B-A3B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3-next-80b-a3b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen/Qwen3-VL-235B-A22B-Instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3.5-397b-a17b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3.5-397b-a17b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3.5-9b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3.5-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwen3.5-plus-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen/Qwen3.6-35B-A3B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen/qwq-32b-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen2.5-32B-EVA-v0.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen25-vl-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3-30b-a3b-instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3-coder-30b-a3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3-max-2026-01-23": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3-vl-235b-a22b-instruct-original": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/qwen3-vl-235b-a22b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3.5-122b-a10b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3.5-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-Anko": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-BlueStar-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-BlueStar-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-BlueStar-v2-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-BlueStar-v2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-BlueStar-v3-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-BlueStar-v3-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-earica-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-earica-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Infracelestial": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-Marvin-DPO-V2-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Marvin-DPO-V2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Marvin-V2-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-Marvin-V2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Musica-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-NaNovel-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-NaNovel-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Omega-Evolution-v2.0-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Omega-Evolution-v2.0-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Omega-Evolution-v2.2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Queen-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-Queen-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-RpRMax-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-Vivid-Durian": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-Writer-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-Writer-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Qwen3.5-27B-Writer-V2-Derestricted": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Qwen3.5-27B-Writer-V2-Derestricted-Lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/qwen3.5-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3.5-omni-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3.5-omni-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3.6-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwen3.7-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/qwq-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/QwQ-32B-ArliAI-RpR-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/raifle/sorcererlm-8x22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/ReadyArt/MS3.2-The-Omega-Directive-24B-Unslop-v2.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/ReadyArt/The-Omega-Abomination-L-70B-v1.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/Salesforce/Llama-xLAM-2-70b-fc-r": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Sao10K/L3-8B-Stheno-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Sao10K/L3.1-70B-Euryale-v2.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Sao10K/L3.1-70B-Hanami-x1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Sao10K/L3.3-70B-Euryale-v2.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/sarvam-105b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/sarvam-30b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/sarvan-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/shisa-ai/shisa-v2-llama3.3-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/shisa-ai/shisa-v2.1-llama3.3-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/sonar": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/sonar-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/sonar-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/sonar-reasoning-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/soob3123/amoral-gemma3-27B-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/soob3123/GrayLine-Qwen3-8B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/soob3123/Veiled-Calla-12B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Steelskull/L3.3-Cu-Mai-R1-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Steelskull/L3.3-Electra-R1-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Steelskull/L3.3-MS-Evalebis-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Steelskull/L3.3-MS-Evayale-70B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Steelskull/L3.3-MS-Nevoria-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Steelskull/L3.3-Nevoria-R1-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/step-2-16k-exp": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/step-2-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/step-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/step-r1-v-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/stepfun-ai/step-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/stepfun-ai/step-3.5-flash-2603": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/stepfun-ai/step-3.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/study_gpt-chatgpt-4o-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/deepseek-r1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/deepseek-v3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/gemma-4-26b-a4b-uncensored": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/gemma4-31b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/glm-5-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/glm-5-1-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/glm-5.1-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/kimi-k2.5-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/llama3-3-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/qwen2.5-vl-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/qwen3-30b-a3b-instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/qwen3-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/qwen3-coder-next": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/qwen3.5-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/qwen3.5-397b-a17b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TEE/qwen3.6-35b-a3b-uncensored": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/tencent/Hunyuan-MT-7B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/tencent/hy3-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Anubis-70B-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Anubis-70B-v1.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Cydonia-24B-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Cydonia-24B-v4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Cydonia-24B-v4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Cydonia-24B-v4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Magidonia-24B-v4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Rocinante-12B-v1.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/Skyfall-31B-v4.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/thedrummer/skyfall-36b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/TheDrummer/UnslopNemo-12B-v4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/THUDM/GLM-4-32B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/THUDM/GLM-4-9B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/THUDM/GLM-Z1-32B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/THUDM/GLM-Z1-9B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/THUDM/GLM-Z1-Rumination-32B-0414": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/tngtech/DeepSeek-TNG-R1T2-Chimera": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/tngtech/tng-r1t-chimera": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/Tongyi-Zhiwen/QwenLong-L1-32B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/undi95/remm-slerp-l2-13b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/universal-summarizer": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/unsloth/gemma-3-12b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/unsloth/gemma-3-1b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/unsloth/gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/unsloth/gemma-3-4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/upstage/solar-pro-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/v0-1.0-md": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/v0-1.5-lg": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/v0-1.5-md": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/venice-uncensored": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/venice-uncensored:web": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/VongolaChouko/Starcannon-Unleashed-12B-v1.0": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/x-ai/grok-4-07-09": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4.1-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4.1-fast-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4.20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4.20-beta-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4.20-beta-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4.20-multi-agent": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4.20-multi-agent-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-build-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/x-ai/grok-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/xiaomi/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/xiaomi/mimo-v2-flash-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/xiaomi/mimo-v2-flash-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/xiaomi/mimo-v2-flash-thinking-original": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nanogpt/xiaomi/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/xiaomi/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/xiaomi/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/xiaomi/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/yi-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/yi-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/yi-medium-200k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/z-ai/glm-4.5v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/z-ai/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/z-ai/glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/z-ai/glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/GLM-4.5-Air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.6-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/GLM-4.6-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.6v-flash-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.6v-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.7-flash-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-4.7-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-5-original": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nanogpt/zai-org/glm-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/abacusai/dracarys-llama-3_1-70b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/bytedance/seed-oss-36b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/deepseek-ai/deepseek-coder-6.7b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/deepseek-ai/deepseek-r1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/deepseek-ai/deepseek-v3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/deepseek-ai/deepseek-v3.1-terminus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/deepseek-ai/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/deepseek-ai/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/deepseek-ai/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/google/gemma-2-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/google/gemma-2-2b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/google/gemma-3-12b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/google/gemma-3-1b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/google/gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/google/gemma-3n-e2b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/google/gemma-3n-e4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/google/gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama-3.1-405b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama-3.1-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama-3.1-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama-3.2-11b-vision-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama-3.2-1b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama-3.2-90b-vision-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama-3.3-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama-4-maverick-17b-128e-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/meta/llama-4-scout-17b-16e-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama3-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/meta/llama3-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/microsoft/phi-3-medium-128k-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/microsoft/phi-3-medium-4k-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/microsoft/phi-3-small-128k-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/microsoft/phi-3-small-8k-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/microsoft/phi-3-vision-128k-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/microsoft/phi-3.5-moe-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/microsoft/phi-3.5-vision-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/microsoft/phi-4-mini-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/minimaxai/minimax-m2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/minimaxai/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/minimaxai/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/minimaxai/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/minimaxai/minimax-m3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/mistralai/codestral-22b-instruct-v0.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/mistralai/devstral-2-123b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/mistralai/ministral-14b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/mistralai/mistral-7b-instruct-v03": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/mistralai/mistral-large-2-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/mistralai/mistral-large-3-675b-instruct-2512": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/mistralai/mistral-medium-3.5-128b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/mistralai/mistral-nemotron": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/mistralai/mistral-small-3.1-24b-instruct-2503": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/mistralai/mistral-small-4-119b-2603": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/mistralai/mixtral-8x22b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/mistralai/mixtral-8x7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/moonshotai/kimi-k2-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/moonshotai/kimi-k2-instruct-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/moonshotai/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/moonshotai/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/moonshotai/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/nvidia/llama-3_3-nemotron-super-49b-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/nvidia/llama-3_3-nemotron-super-49b-v1_5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/nvidia/llama-3.1-nemotron-51b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/nvidia/llama-3.1-nemotron-70b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/nvidia/llama-3.1-nemotron-ultra-253b-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/nvidia/llama3-chatqa-1.5-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/nvidia/mistral-nemo-minitron-8b-8k-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/nvidia/nemotron-3-nano-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "nvidia/nvidia/nemotron-3-super-120b-a12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/nvidia/nemotron-3-ultra-550b-a55b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/nvidia/nemotron-4-340b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/nvidia/nemotron-mini-4b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/nvidia/nemotron-voicechat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/nvidia/nvidia-nemotron-nano-9b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/openai/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/qwen/qwen2.5-coder-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/qwen/qwen2.5-coder-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/qwen/qwen3-235b-a22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/qwen/qwen3-coder-480b-a35b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/qwen/qwen3-next-80b-a3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/qwen/qwen3-next-80b-a3b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/qwen/qwen3.5-122b-a10b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/qwen/qwen3.5-397b-a17b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/sarvamai/sarvam-m": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/stepfun-ai/step-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/stepfun-ai/step-3.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/upstage/solar-10_7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/z-ai/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/z-ai/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/z-ai/glm4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "nvidia/z-ai/glm5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "ollama-cloud/gemma4:31b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "ollama-cloud/gpt-oss:120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "ollama-cloud/gpt-oss:20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "ollama-cloud/qwen3-next:80b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/codex-auto-review": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.3-codex-spark": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai-codex/gpt-image-2": { baseline: true, rationale: "multimodal catalog addition; not yet curated" }, + "openai/codex-mini-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4.1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4.1-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4o-2024-05-13": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4o-2024-08-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4o-2024-11-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.1-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.2-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.3-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.3-codex-spark": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/gpt-image-2": { baseline: true, rationale: "multimodal catalog addition; not yet curated" }, + "openai/gpt-realtime-2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/o1-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/o3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/o3-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/o3-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/o4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openai/o4-mini-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/grok-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/hy3-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/kimi-k2.7-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/kimi-k3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/minimax-m3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/qwen3.5-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/qwen3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/qwen3.7-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-go/qwen3.7-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/big-pickle": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-3-5-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-haiku-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-opus-4-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-opus-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-opus-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-opus-4-7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-opus-4-8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-sonnet-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-sonnet-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-sonnet-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/claude-sonnet-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/deepseek-v4-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gemini-3-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gemini-3-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gemini-3.1-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gemini-3.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gemini-3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.3-codex-spark": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/grok-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/grok-build-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/hy3-preview-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/kimi-k2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/kimi-k2.7-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/laguna-s-2.1-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/ling-2.6-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/mimo-v2-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/mimo-v2-omni-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/mimo-v2-pro-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/mimo-v2.5-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/minimax-m2.5-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/minimax-m3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/nemotron-3-super-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/nemotron-3-ultra-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/north-mini-code-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/qwen3.5-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/qwen3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/qwen3.6-plus-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/ring-2.6-1t-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode-zen/trinity-large-preview-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode/glm-5-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode/gpt-5.3-codex-spark": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opencode/kimi-k2.5-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opengateway/anthropic/claude-sonnet-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opengateway/google/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "opengateway/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~anthropic/claude-fable-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~anthropic/claude-haiku-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~anthropic/claude-opus-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~anthropic/claude-sonnet-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~google/gemini-flash-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~google/gemini-pro-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~moonshotai/kimi-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~openai/gpt-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~openai/gpt-mini-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/~x-ai/grok-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/ai21/jamba-large-1.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/aion-labs/aion-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/aion-labs/aion-3.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/aion-labs/aion-3.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/alibaba/tongyi-deepresearch-30b-a3b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/allenai/olmo-3.1-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/amazon/nova-2-lite-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/amazon/nova-lite-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/amazon/nova-micro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/amazon/nova-premier-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/amazon/nova-pro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-3-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-3.5-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-3.5-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-3.7-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-3.7-sonnet:thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/anthropic/claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-haiku-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4.6-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4.7-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-4.8-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-opus-5-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-sonnet-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-sonnet-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-sonnet-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/anthropic/claude-sonnet-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/arcee-ai/trinity-large-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/arcee-ai/trinity-large-preview:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/arcee-ai/trinity-large-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/arcee-ai/trinity-large-thinking:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/arcee-ai/trinity-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/arcee-ai/trinity-mini:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/arcee-ai/virtuoso-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/auto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/baidu/cobuddy:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/baidu/ernie-4.5-21b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/baidu/ernie-4.5-vl-28b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/bytedance-seed/seed-1.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/bytedance-seed/seed-1.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/bytedance-seed/seed-2.0-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/bytedance-seed/seed-2.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/cohere/command-r-08-2024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/cohere/command-r-plus-08-2024": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/cohere/north-mini-code:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-chat-v3-0324": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-chat-v3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-r1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-r1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-v3.1-terminus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-v3.1-terminus:exacto": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/deepseek/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-v3.2-exp": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-v4-flash:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/deepseek/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/essentialai/rnj-1-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-2.0-flash-001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-2.0-flash-lite-001": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/google/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-2.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-2.5-flash-lite-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/google/gemini-2.5-flash-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/google/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-2.5-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-2.5-pro-preview-05-06": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/google/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-3-pro-image": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-3.1-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-3.1-flash-lite-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/google/gemini-3.1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-3.1-pro-preview-customtools": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/google/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-3.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemini-3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemma-3-12b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemma-3-27b-it:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemma-4-26b-a4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemma-4-26b-a4b-it:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/google/gemma-4-31b-it:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/ibm-granite/granite-4.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inception/mercury": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inception/mercury-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inception/mercury-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inclusionai/ling-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inclusionai/ling-2.6-1t:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inclusionai/ling-2.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inclusionai/ling-2.6-flash:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inclusionai/ring-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/inclusionai/ring-2.6-1t:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/kwaipilot/kat-coder-air-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/kwaipilot/kat-coder-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/kwaipilot/kat-coder-pro-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/kwaipilot/kat-coder-pro-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/liquid/lfm-2.5-1.2b-thinking:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/meituan/longcat-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/meituan/longcat-flash-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/meta-llama/llama-3-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/meta-llama/llama-3.1-405b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/meta-llama/llama-3.1-70b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/meta-llama/llama-3.1-8b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/meta-llama/llama-3.3-70b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/meta-llama/llama-3.3-70b-instruct:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/meta-llama/llama-4-maverick": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/meta-llama/llama-4-scout": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/meta/muse-spark-1.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/meta/muse-spark-1.2": { rationale: "post-rebase catalog addition from dev; not yet curated" }, + "openrouter/minimax/minimax-m1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/minimax/minimax-m2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/minimax/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/minimax/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/minimax/minimax-m2.5:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/minimax/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/minimax/minimax-m3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/codestral-2508": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/devstral-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/devstral-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/devstral-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/ministral-14b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/ministral-3b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/ministral-8b-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-large-2407": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-large-2411": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-large-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-medium-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-medium-3-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-medium-3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-nemo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-saba": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-small-24b-instruct-2501": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/mistralai/mistral-small-2603": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/mistral-small-3.1-24b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/mistralai/mistral-small-3.1-24b-instruct:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/mistralai/mistral-small-3.2-24b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/mistralai/mistral-small-creative": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/mistralai/mixtral-8x22b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/mistralai/mixtral-8x7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/pixtral-large-2411": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/mistralai/voxtral-small-24b-2507": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/moonshotai/kimi-k2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/moonshotai/kimi-k2-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/moonshotai/kimi-k2-0905:exacto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/moonshotai/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/moonshotai/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/moonshotai/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/moonshotai/kimi-k2.6:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/moonshotai/kimi-k2.7-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/moonshotai/kimi-k3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/nex-agi/deepseek-v3.1-nex-n1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/nex-agi/nex-n2-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/nex-agi/nex-n2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/nex-agi/nex-n2-pro:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/nousresearch/deephermes-3-mistral-24b-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nousresearch/hermes-4-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/nvidia/llama-3.1-nemotron-70b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/llama-3.3-nemotron-super-49b-v1.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/nemotron-3-nano-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/nvidia/nemotron-3-nano-30b-a3b:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/nemotron-3-super-120b-a12b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/nemotron-3-super-120b-a12b:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/nemotron-3-ultra-550b-a55b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/nemotron-3-ultra-550b-a55b:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/nemotron-nano-12b-v2-vl:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/nvidia/nemotron-nano-9b-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/nvidia/nemotron-nano-9b-v2:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-3.5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-3.5-turbo-0613": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-3.5-turbo-16k": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4-0314": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4-1106-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4-turbo-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4.1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4.1-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4o-2024-05-13": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4o-2024-08-06": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4o-2024-11-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4o-audio-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4o-mini-2024-07-18": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-4o:extended": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5-image": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5-image-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.1-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.2-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.3-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.6-luna-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.6-sol-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-5.6-terra-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-audio": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-audio-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-oss-120b:exacto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-oss-120b:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-oss-20b:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/gpt-oss-safeguard-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o3-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o3-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o3-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o4-mini-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openai/o4-mini-high": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openrouter/aurora-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openrouter/auto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openrouter/auto-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openrouter/elephant-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openrouter/free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openrouter/healer-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openrouter/hunter-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/openrouter/owl-alpha": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/poolside/laguna-m.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/poolside/laguna-m.1:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/poolside/laguna-s-2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/poolside/laguna-s-2.1:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/poolside/laguna-xs-2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/poolside/laguna-xs-2.1:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/poolside/laguna-xs.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/poolside/laguna-xs.2:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/prime-intellect/intellect-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen-2.5-72b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen-2.5-7b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen-plus-2025-07-28": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen-plus-2025-07-28:thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen-vl-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-14b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-235b-a22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-235b-a22b-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-235b-a22b-thinking-2507": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-30b-a3b-instruct-2507": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-30b-a3b-thinking-2507": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-4b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-4b:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-coder-30b-a3b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-coder-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-coder-next": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-coder-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-coder:exacto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-coder:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-max-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-next-80b-a3b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-next-80b-a3b-instruct:free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-next-80b-a3b-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-vl-235b-a22b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-vl-235b-a22b-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "openrouter/qwen/qwen3-vl-30b-a3b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-vl-30b-a3b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-vl-32b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-vl-8b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3-vl-8b-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.5-122b-a10b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.5-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.5-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.5-397b-a17b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.5-9b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.5-flash-02-23": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.5-plus-02-15": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.5-plus-20260420": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.6-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.6-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.6-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.6-plus-preview:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.6-plus:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.7-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwen3.7-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/qwen/qwq-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/reka/reka-edge": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/rekaai/reka-edge": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/relace/relace-search": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/sakana/fugu-ultra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/sao10k/l3-euryale-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/sao10k/l3.1-euryale-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/stepfun/step-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/stepfun/step-3.5-flash:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/stepfun/step-3.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/tencent/hy3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/tencent/hy3-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/tencent/hy3-preview:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/thedrummer/rocinante-12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/thedrummer/unslopnemo-12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/thinkingmachines/inkling": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/tngtech/deepseek-r1t2-chimera": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/tngtech/tng-r1t-chimera": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/upstage/solar-pro-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/upstage/solar-pro-3:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-3-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-3-mini-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-4-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-4.1-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-4.20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-4.20-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-build-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/x-ai/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/xiaomi/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/xiaomi/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/xiaomi/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/xiaomi/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/xiaomi/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.5-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.5-air:free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.5v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.6:exacto": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "openrouter/z-ai/glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "qianfan/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "qwen-portal/coder-model": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "qwen-portal/vision-model": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:deepseek-ai/DeepSeek-R1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:deepseek-ai/DeepSeek-V3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:deepseek-ai/DeepSeek-V3-0324": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:deepseek-ai/DeepSeek-V3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:meta-llama/Llama-3.3-70B-Instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "synthetic/hf:MiniMaxAI/MiniMax-M2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:MiniMaxAI/MiniMax-M2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:moonshotai/Kimi-K2-Instruct-0905": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "synthetic/hf:moonshotai/Kimi-K2-Thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:moonshotai/Kimi-K2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:moonshotai/Kimi-K2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:nvidia/Kimi-K2.5-NVFP4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "synthetic/hf:openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:Qwen/Qwen3-235B-A22B-Thinking-2507": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "synthetic/hf:Qwen/Qwen3-Coder-480B-A35B-Instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "synthetic/hf:Qwen/Qwen3.5-397B-A17B": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:zai-org/GLM-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:zai-org/GLM-4.7-Flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:zai-org/GLM-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "synthetic/hf:zai-org/GLM-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "together/deepseek-ai/DeepSeek-R1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "together/deepseek-ai/DeepSeek-V3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "together/meta-llama/Llama-3.3-70B-Instruct-Turbo": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "together/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "together/meta-llama/Llama-4-Scout-17B-16E-Instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "together/moonshotai/Kimi-K2-Instruct-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "together/moonshotai/Kimi-K2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "together/zai-org/GLM-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/aion-labs-aion-2-0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/aion-labs-aion-3-0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/aion-labs-aion-3-0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/aion-labs.aion-2-0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/arcee-trinity-large-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-4-6-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-4-7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-4-7-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-4-8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-4-8-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-45": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-opus-5-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-sonnet-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-sonnet-4-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-sonnet-45": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/claude-sonnet-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-gemma-3-27b-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-gemma-4-26b-a4b-uncensored-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-gemma-4-31b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-glm-4-7-flash-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-glm-4-7-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-glm-5-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-glm-5-2-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-gpt-oss-120b-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-gpt-oss-20b-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-qwen-2-5-7b-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-qwen3-30b-a3b-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-qwen3-5-122b-a10b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-qwen3-6-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-qwen3-6-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-qwen3-6-35b-a3b-uncensored-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-qwen3-vl-30b-a3b-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/e2ee-venice-uncensored-24b-p": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/gemini-3-1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/gemini-3-5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/gemini-3-5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/gemini-3-6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/gemma-4-uncensored": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/google-gemma-3-27b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/google-gemma-4-26b-a4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/google-gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/google.gemma-4-26b-a4b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/google.gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-4-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-4-20-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-4-20-multi-agent": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-4-20-multi-agent-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-4-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-4-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-41-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-build-0-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/hermes-3-llama-3.1-405b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/inkling": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/kimi-k2-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/kimi-k2-6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/kimi-k2-7-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/kimi-k3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/llama-3.2-3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/llama-3.3-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/mercury-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/minimax-m21": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/minimax-m25": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/minimax-m27": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/minimax-m3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/minimax-m3-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/mistral-31-24b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/mistral-small-2603": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/mistral-small-3-2-24b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/nvidia-nemotron-3-nano-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/nvidia-nemotron-3-ultra-550b-a55b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/nvidia-nemotron-cascade-2-30b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/olafangensan-glm-4.7-flash-heretic": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-4o-2024-11-20": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-4o-mini-2024-07-18": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-52": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-52-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-53-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-54": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-54-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-54-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-55": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-55-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-56-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-56-luna-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-56-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-56-sol-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-56-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-56-terra-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/openai-gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen-3-6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen-3-7-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen-3-7-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-235b-a22b-instruct-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-235b-a22b-thinking-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-4b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-5-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-5-397b-a17b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-5-9b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-6-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-6-35b-a3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-coder-480b-a35b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-coder-480b-a35b-instruct-turbo": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "venice/qwen3-next-80b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/qwen3-vl-235b-a22b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/venice-uncensored": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/venice-uncensored-1-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/venice-uncensored-role-play": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/xiaomi-mimo-v2-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/z-ai-glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/z-ai-glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/zai-org-glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/zai-org-glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/zai-org-glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/zai-org-glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/zai-org-glm-5-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "venice/zai-org-glm-5-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen-3-14b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen-3-235b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen-3-30b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen-3-32b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen-3.6-max-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-235b-a22b-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3-coder-30b-a3b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-coder-next": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3-coder-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3-max-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-max-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-next-80b-a3b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-next-80b-a3b-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-vl-235b-a22b-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-vl-instruct": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3-vl-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/alibaba/qwen3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3.5-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3.6-27b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3.7-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/alibaba/qwen3.7-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/amazon/nova-2-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/amazon/nova-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/amazon/nova-micro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/amazon/nova-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/anthropic/claude-3-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/anthropic/claude-3.5-haiku": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-3.5-sonnet": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-3.5-sonnet-20240620": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-3.7-sonnet": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/anthropic/claude-haiku-4.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-opus-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/anthropic/claude-opus-4.1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-opus-4.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-opus-4.6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-opus-4.7": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-opus-4.7-fast": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-opus-4.8": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-opus-4.8-fast": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-opus-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/anthropic/claude-opus-5-fast": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-sonnet-4": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-sonnet-4.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-sonnet-4.6": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/anthropic/claude-sonnet-5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/arcee-ai/trinity-large-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/arcee-ai/trinity-large-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/arcee-ai/trinity-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/bytedance/seed-1.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/bytedance/seed-1.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/cohere/command-a": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/deepseek/deepseek-r1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/deepseek/deepseek-v3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/deepseek/deepseek-v3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/deepseek/deepseek-v3.1-terminus": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/deepseek/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/deepseek/deepseek-v3.2-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/deepseek/deepseek-v4-flash": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/deepseek/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/google/gemini-2.0-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/google/gemini-2.0-flash-lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/google/gemini-2.5-flash-lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-2.5-flash-lite-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-2.5-flash-preview-09-2025": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/google/gemini-3-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/google/gemini-3-pro-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-3.1-flash-lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-3.1-flash-lite-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-3.1-pro-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/google/gemini-3.5-flash-lite": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemini-3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/google/gemma-4-26b-a4b-it": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/google/gemma-4-31b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/inception/mercury-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/inception/mercury-coder-small": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/interfaze/interfaze-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/kwaipilot/kat-coder-air-v2.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/kwaipilot/kat-coder-pro-v1": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/kwaipilot/kat-coder-pro-v2": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/kwaipilot/kat-coder-pro-v2.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/meituan/longcat-flash-chat": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/meituan/longcat-flash-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/meta/llama-3.1-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/meta/llama-3.1-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/meta/llama-3.2-11b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/meta/llama-3.2-90b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/meta/llama-3.3-70b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/meta/llama-4-maverick": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/meta/llama-4-scout": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/meta/muse-spark-1.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/minimax/minimax-m2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/minimax/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/minimax/minimax-m2.1-lightning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/minimax/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/minimax/minimax-m2.5-highspeed": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/minimax/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/minimax/minimax-m2.7-highspeed": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/minimax/minimax-m3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/codestral": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/devstral-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/devstral-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/devstral-small-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/magistral-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/magistral-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/ministral-14b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/ministral-3b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/ministral-8b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/mistral-large-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/mistral-medium": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/mistral-medium-3.5": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/mistral/mistral-nemo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/mistral-small": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/pixtral-12b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/mistral/pixtral-large": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/moonshotai/kimi-k2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/moonshotai/kimi-k2-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/moonshotai/kimi-k2-thinking": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/moonshotai/kimi-k2-thinking-turbo": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/moonshotai/kimi-k2-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/moonshotai/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/moonshotai/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/moonshotai/kimi-k2.7-code": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/moonshotai/kimi-k2.7-code-highspeed": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/moonshotai/kimi-k3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/nvidia/nemotron-3-nano-30b-a3b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/nvidia/nemotron-3-super-120b-a12b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/nvidia/nemotron-3-ultra-550b-a55b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/nvidia/nemotron-nano-12b-v2-vl": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/nvidia/nemotron-nano-9b-v2": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/openai/codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-3.5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-4-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-4.1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-4.1-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.1-codex-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.1-codex-mini": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/openai/gpt-5.1-instant": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.1-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.2-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.3-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-oss-120b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-oss-20b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/gpt-oss-safeguard-20b": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/openai/o1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/o3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/o3-deep-research": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/o3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/o3-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/openai/o4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/perplexity/sonar": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/perplexity/sonar-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/poolside/laguna-s-2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/poolside/laguna-s-2.1-free": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/prime-intellect/intellect-3": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/sakana/fugu-ultra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/stepfun/step-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/stepfun/step-3.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/thinkingmachines/inkling": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/vercel/v0-1.0-md": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/vercel/v0-1.5-md": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-2-vision": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-3-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-3-mini-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-4-fast-non-reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4-fast-reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.1-fast-non-reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.1-fast-reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.20-multi-agent": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.20-multi-agent-beta": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.20-non-reasoning": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.20-non-reasoning-beta": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.20-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-4.20-reasoning-beta": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "vercel-ai-gateway/xai/grok-4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-build-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xai/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xiaomi/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xiaomi/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xiaomi/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/xiaomi/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.5-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.5v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.6v-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-4.7-flashx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-5.2-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "vercel-ai-gateway/zai/glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-2-1212": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-2-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-2-vision": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-2-vision-1212": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-2-vision-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-3-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-3-fast-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-3-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-3-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-3-mini-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-3-mini-fast-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-3-mini-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4-1-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4-1-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4.20-0309-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4.20-0309-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4.20-beta-latest-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4.20-beta-latest-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-4.20-multi-agent-beta-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-build-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-composer-2.5-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xai/grok-vision-beta": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-ams/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-ams/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-ams/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-ams/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-ams/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-ams/mimo-v2.5-pro-ultraspeed": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "xiaomi-token-plan-cn/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-cn/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-cn/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-cn/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-cn/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-cn/mimo-v2.5-pro-ultraspeed": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "xiaomi-token-plan-sgp/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-sgp/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-sgp/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-sgp/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-sgp/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi-token-plan-sgp/mimo-v2.5-pro-ultraspeed": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "xiaomi/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "xiaomi/mimo-v2.5-pro-ultraspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.5-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.5v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-4.7-flashx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zai/glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-3.5-haiku": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-3.5-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-3.7-sonnet": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-fable-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-haiku-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-opus-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-opus-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-opus-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-opus-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-opus-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-opus-4.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-sonnet-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-sonnet-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-sonnet-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-sonnet-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/anthropic/claude-sonnet-5-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/baidu/ernie-5.0-thinking-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/baidu/ernie-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/baidu/ernie-x1.1-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/bytedance/doubao-seed-1.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/bytedance/doubao-seed-2.0-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/bytedance/doubao-seed-2.0-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/bytedance/doubao-seed-2.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/bytedance/doubao-seed-2.0-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/bytedance/doubao-seed-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-chat-v3.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-r1-0528": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-reasoner": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-v3.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-v3.2-exp": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-v4-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-v4-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-v4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/deepseek/deepseek-v4-pro-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-2.0-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-2.0-flash-lite-001": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-2.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-2.5-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-3-flash-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-3-pro-image-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-3-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-3.1-flash-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-3.1-flash-lite-preview": { + baseline: true, + rationale: "pre-feature baseline; not yet curated", + }, + "zenmux/google/gemini-3.1-pro-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemini-3.5-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/google/gemma-3-12b-it": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ling-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ling-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ling-2.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ling-flash-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ling-mini-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/llada2.0-flash-cap": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/llada2.1-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ming-flash-omni-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ring-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ring-2.6-1t": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ring-flash-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/inclusionai/ring-mini-2.0": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/kuaishou/kat-coder-pro-v1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/kuaishou/kat-coder-pro-v1-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/kuaishou/kat-coder-pro-v2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/meta/llama-3.3-70b-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/meta/llama-4-scout-17b-16e-instruct": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/minimax/minimax-m2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/minimax/minimax-m2-her": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/minimax/minimax-m2.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/minimax/minimax-m2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/minimax/minimax-m2.5-lightning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/minimax/minimax-m2.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/minimax/minimax-m2.7-highspeed": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/minimax/minimax-m3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/mistralai/mistral-large-2512": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k2-0711": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k2-0905": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k2-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k2-thinking-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k2.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k2.7-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k2.7-code-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/moonshotai/kimi-k3-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/chat-latest": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-4.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-4.1-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-4.1-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-4o": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-4o-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.1-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.1-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.1-codex-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.2-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.2-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.3-chat": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.3-codex": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.4-nano": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.4-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.5-instant": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.6-luna": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.6-sol": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/gpt-5.6-terra": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/openai/o4-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3-14b": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3-235b-a22b-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3-235b-a22b-thinking-2507": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3-coder": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3-coder-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3-vl-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3.5-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3.6-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3.6-max-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3.6-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3.7-max": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/qwen/qwen3.7-plus": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/sapiens-ai/agnes-1.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/sapiens-ai/agnes-1.5-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/sapiens-ai/agnes-1.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/sapiens-ai/agnes-2.0-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/stepfun/step-3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/stepfun/step-3.5-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/stepfun/step-3.5-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/stepfun/step-3.7-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/stepfun/step-3.7-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/tencent/hunyuan-2.0-thinking": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/tencent/hy3-preview": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/volcengine/doubao-seed-1-6-vision": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/volcengine/doubao-seed-1.8": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/volcengine/doubao-seed-2.0-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/volcengine/doubao-seed-2.0-lite": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/volcengine/doubao-seed-2.0-mini": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/volcengine/doubao-seed-2.0-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/volcengine/doubao-seed-code": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4.1-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4.1-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4.2-fast": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4.2-fast-non-reasoning": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4.3": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-build-0.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/x-ai/grok-code-fast-1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/xiaomi/mimo-v2-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/xiaomi/mimo-v2-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/xiaomi/mimo-v2-omni": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/xiaomi/mimo-v2-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/xiaomi/mimo-v2.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/xiaomi/mimo-v2.5-pro": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.5-air": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.6": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.6v": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.6v-flash": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.6v-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.7": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.7-flash-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-4.7-flashx": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-5": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-5-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-5.1": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-5.2": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-5.2-free": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, + "zenmux/z-ai/glm-5v-turbo": { baseline: true, rationale: "pre-feature baseline; not yet curated" }, +} as const satisfies TierMapSkipList; + +export const CURATED_TIER_MAP: AutoroutingCuratedTierMap = { + labels: CURATED_TIER_LABELS, + skips: TIER_MAP_SKIP_LIST, + skipList: TIER_MAP_SKIP_LIST, + version: TIER_MAP_VERSION, +}; + +/** Canonical JSON uses sorted object keys and preserves array order. */ +export function canonicalJson(value: unknown): string { + if (value === null) return "null"; + if (value === undefined) return "undefined"; + switch (typeof value) { + case "string": + case "boolean": + return JSON.stringify(value); + case "number": + return Number.isFinite(value) ? JSON.stringify(value) : "null"; + case "bigint": + throw new TypeError("Cannot canonicalize bigint"); + case "function": + case "symbol": + return "undefined"; + case "object": + if (Array.isArray(value)) { + return `[${value + .map(item => { + const encoded = canonicalJson(item); + return encoded === "undefined" ? "null" : encoded; + }) + .join(",")}]`; + } + return `{${Object.keys(value as Record) + .sort() + .flatMap(key => { + const encoded = canonicalJson((value as Record)[key]); + return encoded === "undefined" ? [] : [`${JSON.stringify(key)}:${encoded}`]; + }) + .join(",")}}`; + default: + return "undefined"; + } +} + +/** UTF-8 bytes for the one-line canonical JSON representation. */ +export function canonicalJsonBytes(value: unknown): Uint8Array { + return new TextEncoder().encode(canonicalJson(value)); +} + +function sha256Canonical(value: unknown): string { + return createHash("sha256").update(canonicalJsonBytes(value)).digest("hex"); +} + +function asCuratedMap(input: AutoroutingCuratedTierMap | CuratedTierLabels): NormalizedAutoroutingCuratedTierMap { + if ("labels" in input && input.labels !== undefined) { + return { labels: input.labels, skips: input.skips ?? input.skipList ?? {}, version: input.version }; + } + return { labels: input, skips: {}, version: TIER_MAP_VERSION }; +} + +/** Return the fingerprint of labels, skips, and the curation map version. */ +export function computeMapFingerprint( + input: AutoroutingCuratedTierMap | CuratedTierLabels = CURATED_TIER_MAP, + skips?: TierMapSkipList, + version?: number, +): string { + const map = + skips === undefined && version === undefined + ? asCuratedMap(input) + : { + labels: ("labels" in input ? input.labels : input) as CuratedTierLabels, + skips: skips ?? {}, + version: version ?? TIER_MAP_VERSION, + }; + return sha256Canonical({ labels: map.labels, skips: map.skips, version: map.version }); +} + +export type TierMapValidationIssue = { + path: string; + message: string; +}; + +const KEY_PATTERN = /^[^/\s*?[\]]+\/[^\s*?[\]]+$/u; +const EFFORTS = new Set(["minimal", "low", "medium", "high", "xhigh"]); +const TIER_SET = new Set(AUTOROUTING_TIERS); + +function defaultCatalog(): readonly Model[] { + return getBundledProviders().flatMap(provider => + getBundledModels(provider as Parameters[0]), + ); +} + +function modelKey(model: Pick): string { + return `${model.provider}/${model.id}`; +} + +function providerFromKey(key: string): string { + return key.slice(0, key.indexOf("/")); +} + +/** Collect structural and catalog-aware curation errors without throwing. */ +export function collectTierMapValidationIssues( + input: AutoroutingCuratedTierMap | CuratedTierLabels, + catalog: readonly Model[] = defaultCatalog(), +): TierMapValidationIssue[] { + const map = asCuratedMap(input); + const issues: TierMapValidationIssue[] = []; + if (!Number.isInteger(map.version) || map.version < 1) { + issues.push({ path: "version", message: "Expected a positive integer map version." }); + } + if (map.labels === null || typeof map.labels !== "object" || Array.isArray(map.labels)) { + issues.push({ path: "labels", message: "Expected a labels object." }); + } + if (map.skips === null || typeof map.skips !== "object" || Array.isArray(map.skips)) { + issues.push({ path: "skips", message: "Expected a skips object." }); + } + const catalogByKey = new Map(catalog.map(model => [modelKey(model), model])); + const ranks = new Map(); + const labelKeys = new Set(); + for (const [key, assignments] of Object.entries(map.labels ?? {})) { + labelKeys.add(key); + if (!KEY_PATTERN.test(key) || !isValidAutoroutingSelector(key)) { + issues.push({ + path: `labels.${key}`, + message: `Invalid provider/model key; expected ${AUTOROUTING_SELECTOR_PATTERN}.`, + }); + continue; + } + const catalogModel = catalogByKey.get(key); + if (!catalogModel) { + issues.push({ path: `labels.${key}`, message: "Labeled model is absent from the committed catalog." }); + } + if (!Array.isArray(assignments)) { + issues.push({ path: `labels.${key}`, message: "Expected an assignment array." }); + continue; + } + const tiers = new Set(); + for (let index = 0; index < assignments.length; index += 1) { + const assignment = assignments[index] as Partial | null; + const path = `labels.${key}.${index}`; + if (assignment === null || typeof assignment !== "object") { + issues.push({ path, message: "Expected an assignment object." }); + continue; + } + if (!TIER_SET.has(assignment.tier as AutoroutingTier)) { + issues.push({ path: `${path}.tier`, message: "Unknown autorouting tier." }); + } else if (tiers.has(assignment.tier as AutoroutingTier)) { + issues.push({ path, message: "A model may have at most one assignment per tier." }); + } else { + tiers.add(assignment.tier as AutoroutingTier); + } + const rank = assignment.rank; + if (typeof rank !== "number" || !Number.isInteger(rank) || rank < 1) { + issues.push({ path: `${path}.rank`, message: "Rank must be a positive integer." }); + } else { + const rankKey = `${providerFromKey(key)}\u0000${assignment.tier}`; + const prior = ranks.get(`${rankKey}\u0000${rank}`); + if (prior !== undefined && prior !== key) { + issues.push({ path: `${path}.rank`, message: `Rank collides with ${prior} for provider/tier.` }); + } else { + ranks.set(`${rankKey}\u0000${rank}`, key); + } + } + if (assignment.effort !== undefined) { + if (!EFFORTS.has(assignment.effort as TierEffort)) { + issues.push({ path: `${path}.effort`, message: "Unknown tier effort." }); + } else if (catalogModel && !catalogModel.reasoning) { + issues.push({ + path: `${path}.effort`, + message: "Effort is only valid for reasoning-capable catalog models.", + }); + } + } + } + } + for (const [key, skip] of Object.entries(map.skips ?? {})) { + if (!KEY_PATTERN.test(key) || !isValidAutoroutingSelector(key)) { + issues.push({ path: `skips.${key}`, message: "Invalid provider/model key." }); + } + if (labelKeys.has(key)) + issues.push({ path: `skips.${key}`, message: "A key cannot be both labeled and skipped." }); + if (skip === null || typeof skip !== "object" || Array.isArray(skip)) { + issues.push({ path: `skips.${key}`, message: "Expected a skip descriptor." }); + continue; + } + if (typeof skip.rationale !== "string" || skip.rationale.trim() === "") { + issues.push({ path: `skips.${key}.rationale`, message: "Skip rationale must be a non-empty string." }); + } + if (skip.baseline !== undefined && skip.baseline !== true) { + issues.push({ path: `skips.${key}.baseline`, message: "Skip baseline, when present, must be true." }); + } + } + return issues; +} + +/** Validate a curation map at module load or a caller-provided test boundary. */ +export function validateTierMap( + input: AutoroutingCuratedTierMap | CuratedTierLabels, + catalog?: readonly Model[], +): void { + const issues = collectTierMapValidationIssues(input, catalog); + if (issues.length > 0) { + throw new Error( + `Invalid autorouting tier map:\n${issues.map(issue => `- ${issue.path}: ${issue.message}`).join("\n")}`, + ); + } +} + +export const validateCuratedTierMap = validateTierMap; +export const assertValidTierMap = validateTierMap; + +validateTierMap(CURATED_TIER_MAP); diff --git a/packages/coding-agent/src/config/autorouting.ts b/packages/coding-agent/src/config/autorouting.ts new file mode 100644 index 0000000000..563d1c0dde --- /dev/null +++ b/packages/coding-agent/src/config/autorouting.ts @@ -0,0 +1,120 @@ +import type { Model } from "@gajae-code/ai/core"; +import { splitSelectorThinkingSuffix } from "../thinking"; +import { type AutoroutingEffective, type AutoroutingTier, DEFAULT_AUTOROUTING_TIER } from "./autorouting-contract"; +import { formatModelString } from "./model-resolver"; + +export type TierSelectorNormalization = + | { pinned: string } + | { rejected: "selector_not_provider_qualified" } + | { unmatched: true }; + +export type RoutingOutcome = + | { kind: "disabled" } + | { + kind: "routed"; + tier: AutoroutingTier; + requestedTier?: AutoroutingTier; + defaultTierApplied?: true; + pinnedSelector: string; + } + | { + kind: "manual-fallback"; + tier: AutoroutingTier; + requestedTier?: AutoroutingTier; + defaultTierApplied?: true; + attemptedSelectorCount: number; + reason: "tier_unmatched" | "tier_missing_in_map"; + }; + +function isProviderQualified(selector: string): boolean { + const slash = selector.indexOf("/"); + if (slash <= 0 || slash === selector.length - 1) return false; + if (/[*?[]/.test(selector)) return false; + if (selector.slice(0, slash).toLowerCase() === "pi") return false; + return selector.trim() === selector; +} + +/** + * Pin one configured selector against exactly the supplied ordered snapshot. + * No settings, role aliases, usage ordering, or fuzzy matching participate. + */ +export function normalizeTierSelector(selector: string, snapshot: readonly Model[]): TierSelectorNormalization { + if (!isProviderQualified(selector)) return { rejected: "selector_not_provider_qualified" }; + const slash = selector.indexOf("/"); + const provider = selector.slice(0, slash); + const rest = selector.slice(slash + 1); + + // Literal first: colon-bearing model ids are preserved when they exist. + const literal = snapshot.find( + model => model.provider.toLowerCase() === provider.toLowerCase() && model.id.toLowerCase() === rest.toLowerCase(), + ); + if (literal) return { pinned: formatModelString(literal) }; + + const suffix = splitSelectorThinkingSuffix(rest); + if (suffix.invalidSuffix !== undefined) return { unmatched: true }; + if (suffix.thinkingLevel === undefined) return { unmatched: true }; + const baseId = suffix.selector; + const model = snapshot.find( + candidate => + candidate.provider.toLowerCase() === provider.toLowerCase() && + candidate.id.toLowerCase() === baseId.toLowerCase(), + ); + if (!model) return { unmatched: true }; + return { pinned: `${formatModelString(model)}:${suffix.thinkingLevel}` }; +} + +export function resolveTaskRouting(input: { + effectiveAutorouting: AutoroutingEffective; + requestedTier?: AutoroutingTier; + availableModels?: readonly Model[]; +}): RoutingOutcome { + const { effectiveAutorouting, requestedTier, availableModels } = input; + if (!effectiveAutorouting.active) return { kind: "disabled" }; + + const tier = requestedTier ?? DEFAULT_AUTOROUTING_TIER; + const defaultTierApplied = requestedTier === undefined ? true : undefined; + const selectors = effectiveAutorouting.map[tier]; + if (!selectors || selectors.length === 0) { + return { + kind: "manual-fallback", + tier, + requestedTier, + ...(defaultTierApplied ? { defaultTierApplied } : {}), + attemptedSelectorCount: 0, + reason: "tier_missing_in_map", + }; + } + if (!availableModels) { + return { + kind: "manual-fallback", + tier, + requestedTier, + ...(defaultTierApplied ? { defaultTierApplied } : {}), + attemptedSelectorCount: selectors.length, + reason: "tier_unmatched", + }; + } + + let attemptedSelectorCount = 0; + for (const selector of selectors) { + attemptedSelectorCount++; + const normalized = normalizeTierSelector(selector, availableModels); + if ("pinned" in normalized) { + return { + kind: "routed", + tier, + requestedTier, + ...(defaultTierApplied ? { defaultTierApplied } : {}), + pinnedSelector: normalized.pinned, + }; + } + } + return { + kind: "manual-fallback", + tier, + requestedTier, + ...(defaultTierApplied ? { defaultTierApplied } : {}), + attemptedSelectorCount, + reason: "tier_unmatched", + }; +} diff --git a/packages/coding-agent/src/config/model-registry.ts b/packages/coding-agent/src/config/model-registry.ts index 7571adda25..c69e213802 100644 --- a/packages/coding-agent/src/config/model-registry.ts +++ b/packages/coding-agent/src/config/model-registry.ts @@ -89,6 +89,7 @@ import { createProviderSelectionPolicy, type EffectiveProviderAuth, type ProviderSelectionPolicy, + projectCatalogProviderOrder, } from "./provider-selection-policy"; import { type Settings, settings } from "./settings"; @@ -3745,6 +3746,23 @@ export class ModelRegistry { }); } + /** + * Deterministic provider priority for autorouting tier generation: configured + * `modelProviderOrder` first, then first-wins catalog order. + * + * Deliberately takes no session and never touches `authStorage`. It bypasses + * `#buildProviderSelectionPolicy` entirely so no `effectiveAuth` map is even + * assembled — auth-independence is structural here, not a convention. Ranking + * that *is* auth-aware stays private to the policy. + * + * Providers absent from the catalog are dropped so a dead declaration cannot + * pollute the generated setup's `declarationFingerprint`. Returned ids use the + * catalog's original spelling because the generator matches provider prefixes + * with case-sensitive exact strings. + */ + autoroutingProviderOrder(): readonly string[] { + return projectCatalogProviderOrder(getConfiguredProviderOrderFromSettings(), this.#models); + } #providerRankMap(policy: ProviderSelectionPolicy): Map { const providerRank = new Map(); for (const provider of policy.orderedProviders()) { diff --git a/packages/coding-agent/src/config/provider-selection-policy.ts b/packages/coding-agent/src/config/provider-selection-policy.ts index 28e356bf6c..470e17dff3 100644 --- a/packages/coding-agent/src/config/provider-selection-policy.ts +++ b/packages/coding-agent/src/config/provider-selection-policy.ts @@ -81,15 +81,8 @@ export function createProviderSelectionPolicy(input: ProviderSelectionPolicyInpu } } - const orderedProviders = [...explicitProviders]; - const seen = new Set(explicitSet); - for (const provider of input.catalogProviders) { - if (seen.has(provider)) { - continue; - } - seen.add(provider); - orderedProviders.push(provider); - } + // One shared implementation with the standalone accessor; see projectProviderOrder. + const orderedProviders = projectProviderOrder(explicitProviders, input.catalogProviders); return { rank(provider: string): number { @@ -113,6 +106,35 @@ export function createProviderSelectionPolicy(input: ProviderSelectionPolicyInpu }; } +/** + * Project the deterministic provider order: normalized explicit order first, then + * first-wins catalog order for everything else. + * + * This is the single implementation of that ordering. It reads no credentials and + * takes no auth input at all, so any consumer that only needs "which providers, in + * what priority" cannot accidentally acquire auth sensitivity. Auth-aware banding + * lives exclusively in {@link ProviderSelectionPolicy.rank}. + */ +export function projectProviderOrder( + explicitProviderOrder: readonly string[], + catalogProviders: readonly string[], +): string[] { + const ordered: string[] = []; + const seen = new Set(); + for (const raw of explicitProviderOrder) { + const normalized = raw.trim().toLowerCase(); + if (!normalized || seen.has(normalized)) continue; + seen.add(normalized); + ordered.push(normalized); + } + for (const provider of catalogProviders) { + if (!provider || seen.has(provider)) continue; + seen.add(provider); + ordered.push(provider); + } + return ordered; +} + export interface ProviderSelectionCatalog { /** Lowercased provider ids in first-wins registry catalog order. */ readonly catalogProviders: readonly string[]; @@ -144,3 +166,32 @@ export function buildProviderSelectionCatalog(models: readonly Model[]): Pr } return { catalogProviders, catalogModels }; } +/** + * Deterministic provider priority for a catalog, returned in the catalog's own + * spelling. + * + * Ordering and de-duplication run on normalized ids, but the result restores each + * provider's first-seen catalog spelling because the autorouting generator matches + * provider prefixes with case-sensitive exact strings — a lowercased id would + * silently empty that provider's tiers. Providers absent from the catalog are + * dropped so a dead declaration cannot pollute a generated declarationFingerprint. + * + * Reads no credentials: it takes a catalog and an explicit order, nothing else. + */ +export function projectCatalogProviderOrder( + explicitProviderOrder: readonly string[], + models: readonly Model[], +): string[] { + const { catalogProviders } = buildProviderSelectionCatalog(models); + const spelling = new Map(); + for (const model of models) { + const normalized = model.provider.trim().toLowerCase(); + if (normalized && !spelling.has(normalized)) spelling.set(normalized, model.provider); + } + const restored: string[] = []; + for (const provider of projectProviderOrder(explicitProviderOrder, catalogProviders)) { + const spelled = spelling.get(provider); + if (spelled !== undefined) restored.push(spelled); + } + return restored; +} diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index 0dc66e24a2..c944653788 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -4,6 +4,18 @@ import { TASK_SIMPLE_MODES } from "../task/simple-mode"; import { getThinkingLevelMetadata } from "../thinking-metadata"; import { DEFAULT_EDIT_MODE_SETTING, EDIT_MODE_SETTINGS, EDIT_MODES, type EditMode } from "../utils/edit-mode"; import { CONFIGURABLE_SEARCH_PROVIDER_IDS } from "../web/search/types"; +import { + AUTOROUTING_SELECTOR_DESCRIPTION, + AUTOROUTING_SELECTOR_PATTERN, + AUTOROUTING_TIERS, + type AutoroutingLocalIssue, + type AutoroutingProvenance, + type AutoroutingSetup, + type AutoroutingTierMapInput, + validateAutoroutingLocal, + validateAutoroutingProvenance, + validateAutoroutingSetup, +} from "./autorouting-contract"; import type { ModelSelectorValue } from "./model-selector-value"; import { UPDATE_CHANNELS } from "./update-channel"; @@ -148,6 +160,22 @@ export type AnyUiMetadata = UiBase & { options?: ReadonlyArray | "runtime"; }; +/** JSON Schema fragment carried by settings definitions that own nested validation. */ +export type JsonSchemaObject = { + [key: string]: unknown; + type?: string; + properties?: Record; + additionalProperties?: boolean | JsonSchemaObject; + items?: JsonSchemaObject; + required?: readonly string[]; + pattern?: string; + minItems?: number; + minLength?: number; + uniqueItems?: boolean; + minimum?: number; + const?: unknown; +}; + interface BooleanDef { type: "boolean"; default?: boolean; @@ -186,6 +214,21 @@ type RecordValueDef = | { type: "string-enum"; values: readonly string[] } | { type: "credential-selector" }; +interface ConstrainedRecordValueDef { + type: "autorouting-selector-value"; + pattern: string; + description: string; +} + +interface ConstrainedRecordDef { + type: "constrained-record"; + default: T; + keys: readonly string[]; + valueSchema: ConstrainedRecordValueDef; + description?: string; + ui?: UiBase; +} + interface RecordDef { type: "record"; default: Record; @@ -193,13 +236,23 @@ interface RecordDef { ui?: UiBase; } -type SettingDef = +export interface OptionalObjectDef { + type: "optional-object"; + default: undefined; + jsonSchema: JsonSchemaObject; + validate: (value: unknown) => AutoroutingLocalIssue[]; + _value?: T; +} + +export type SettingDef = | BooleanDef | StringDef | NumberDef | EnumDef | ArrayDef - | RecordDef; + | RecordDef + | ConstrainedRecordDef + | OptionalObjectDef; // ═══════════════════════════════════════════════════════════════════════════ // Schema Definition @@ -261,6 +314,52 @@ export const DEFAULT_BASH_INTERCEPTOR_RULES: BashInterceptorRule[] = [ }, ]; +const AUTOROUTING_SETUP_JSON_SCHEMA: JsonSchemaObject = { + type: "object", + properties: { + schema: { type: "integer", const: 1 }, + providers: { + type: "array", + minItems: 1, + uniqueItems: true, + items: { type: "string", minLength: 1 }, + }, + models: { + type: "array", + items: { + type: "string", + minLength: 1, + maxLength: 256, + pattern: AUTOROUTING_SELECTOR_PATTERN, + not: { pattern: "^\\s*[pP][iI]/" }, + }, + }, + }, + additionalProperties: false, + required: ["schema", "providers"], +}; + +const AUTOROUTING_PROVENANCE_JSON_SCHEMA: JsonSchemaObject = { + type: "object", + properties: { + schema: { type: "integer", const: 1 }, + source: { + type: "object", + properties: { + catalogFingerprint: { type: "string", pattern: "^[0-9a-f]{64}$" }, + mapFingerprint: { type: "string", pattern: "^[0-9a-f]{64}$" }, + generatorVersion: { type: "integer", minimum: 1 }, + }, + additionalProperties: false, + required: ["catalogFingerprint", "mapFingerprint", "generatorVersion"], + }, + declarationFingerprint: { type: "string", pattern: "^[0-9a-f]{64}$" }, + tiersFingerprint: { type: "string", pattern: "^[0-9a-f]{64}$" }, + }, + additionalProperties: false, + required: ["schema", "source", "declarationFingerprint", "tiersFingerprint"], +}; + export const SETTINGS_SCHEMA = { // ──────────────────────────────────────────────────────────────────────── // General settings (no UI) @@ -3510,6 +3609,39 @@ export const SETTINGS_SCHEMA = { valueSchema: MODEL_SELECTOR_VALUE_SCHEMA, }, + "task.autorouting.enabled": { + type: "boolean", + default: false, + }, + + "task.autorouting.tiers": { + type: "constrained-record", + default: {} as AutoroutingTierMapInput, + keys: AUTOROUTING_TIERS, + valueSchema: { + type: "autorouting-selector-value", + minLength: 1, + maxLength: 256, + pattern: AUTOROUTING_SELECTOR_PATTERN, + description: AUTOROUTING_SELECTOR_DESCRIPTION, + }, + description: AUTOROUTING_SELECTOR_DESCRIPTION, + }, + + "task.autorouting.setup": { + type: "optional-object", + default: undefined, + jsonSchema: AUTOROUTING_SETUP_JSON_SCHEMA, + validate: validateAutoroutingSetup, + } as OptionalObjectDef, + + "task.autorouting.provenance": { + type: "optional-object", + default: undefined, + jsonSchema: AUTOROUTING_PROVENANCE_JSON_SCHEMA, + validate: validateAutoroutingProvenance, + } as OptionalObjectDef, + "tasks.todoClearDelay": { type: "number", default: 60, @@ -3856,7 +3988,11 @@ export type SettingValue

= Schema[P] extends { type: "boo ? D : Schema[P] extends { type: "record"; default: infer D } ? D - : never; + : Schema[P] extends { type: "constrained-record"; default: infer D } + ? D + : Schema[P] extends OptionalObjectDef + ? D | undefined + : never; /** Get the default value for a setting path */ export function getDefault

(path: P): SettingValue

{ @@ -3930,7 +4066,11 @@ function schemaPaths(value: Record, prefix = ""): string[] { const path = prefix ? `${prefix}.${key}` : key; const definition = SETTINGS_SCHEMA[path as SettingPath]; // Records intentionally accept user-defined keys; validate their entries below. - if (definition?.type === "record") { + if ( + definition?.type === "record" || + definition?.type === "constrained-record" || + definition?.type === "optional-object" + ) { paths.push(path); } else if (child && typeof child === "object" && !Array.isArray(child)) { paths.push(...schemaPaths(child as Record, path)); @@ -3974,7 +4114,15 @@ function validSettingValue(definition: (typeof SETTINGS_SCHEMA)[SettingPath], va (definition.values as readonly string[]).includes(value)) || (definition.type === "array" && validArraySettingValue(value, "items" in definition ? definition.items?.enum : undefined)) || - (definition.type === "record" && !!value && typeof value === "object" && !Array.isArray(value)) + ((definition.type === "record" || definition.type === "constrained-record") && + !!value && + typeof value === "object" && + !Array.isArray(value)) || + (definition.type === "optional-object" && + !!value && + typeof value === "object" && + !Array.isArray(value) && + definition.validate(value).length === 0) ); } @@ -4022,6 +4170,16 @@ export function validateSettingPatch(patch: Record): Array<{ pa issues.push({ path, detail }); continue; } + if (definition.type === "constrained-record" && path === "task.autorouting.tiers") { + for (const issue of validateAutoroutingLocal({ tiers: value })) { + const nestedPath = issue.path.replace(/^tiers\./u, ""); + issues.push({ + path: nestedPath ? `task.autorouting.tiers.${nestedPath}` : "task.autorouting.tiers", + detail: issue.detail, + }); + } + continue; + } if (definition.type === "record" && "valueSchema" in definition && definition.valueSchema) { for (const [key, entry] of Object.entries(value as Record)) { if (!validRecordValue(definition.valueSchema, entry)) { @@ -4065,6 +4223,33 @@ export function reconcileSettingsSchema(raw: Record): { schemaSetAtPath(settings, path, next); issues.push({ path, kind: "coerced", detail: `Coerced ${typeof value} to ${definition.type}.` }); } + if (definition.type === "optional-object") { + for (const localIssue of definition.validate(next)) { + issues.push({ + path: localIssue.path ? `${path}.${localIssue.path}` : path, + kind: "invalid", + detail: localIssue.detail, + }); + } + continue; + } + if (definition.type === "constrained-record") { + const autorouting = schemaValueAtPath(settings, "task.autorouting"); + const tiersFragment = + autorouting && typeof autorouting === "object" && !Array.isArray(autorouting) + ? Object.fromEntries( + Object.entries(autorouting).filter(([key]) => key !== "setup" && key !== "provenance"), + ) + : autorouting; + for (const localIssue of validateAutoroutingLocal(tiersFragment)) { + issues.push({ + path: localIssue.path ? `task.autorouting.${localIssue.path}` : "task.autorouting", + kind: "invalid", + detail: localIssue.detail, + }); + } + continue; + } if (!validSettingValue(definition, next)) { const arrayItemEnum = definition.type === "array" && Array.isArray(next) && "items" in definition diff --git a/packages/coding-agent/src/config/settings.ts b/packages/coding-agent/src/config/settings.ts index df24bd8dae..76e7418c5a 100644 --- a/packages/coding-agent/src/config/settings.ts +++ b/packages/coding-agent/src/config/settings.ts @@ -57,6 +57,11 @@ import { setByPath, withAtomicYamlConfigTransaction, } from "./atomic-yaml-patch"; +import { + type AutoroutingEffective, + validateAutoroutingEffective, + validateAutoroutingLocal, +} from "./autorouting-contract"; import { isModelSelectorValue, type ModelSelectorValue, normalizeModelSelectorValue } from "./model-selector-value"; import { @@ -68,6 +73,7 @@ import { reconcileSettingsSchema, SETTINGS_SCHEMA, type SettingPath, + type SettingsSchemaIssue, type SettingsSchemaReport, type SettingValue, } from "./settings-schema"; @@ -524,6 +530,9 @@ export class Settings implements NotificationSettingsReader { #legacyCustomImageProviderDiagnosticLogged = false; #schemaReport: SettingsSchemaReport = { issues: [], valid: true }; + + #autoroutingEffective: AutoroutingEffective = { active: false }; + #autoroutingLocalIssues: SettingsSchemaIssue[] = []; #schemaMigrationPending = false; /** A newer config schema must never be rewritten by legacy migrations. */ #futureSchemaVersion = false; @@ -692,7 +701,34 @@ export class Settings implements NotificationSettingsReader { /** Diagnostics from schema reconciliation during the most recent load. */ getSchemaReport(): SettingsSchemaReport { - return structuredClone(this.#schemaReport); + const issues = [ + ...this.#schemaReport.issues.filter( + issue => + !( + (issue.kind === "invalid" || issue.kind === "unknown") && + (issue.path === "task.autorouting" || issue.path.startsWith("task.autorouting.")) + ), + ), + ...this.#autoroutingLocalIssues, + ...(this.#autoroutingEffective.active || !this.#autoroutingEffective.issue + ? [] + : [ + { + path: "task.autorouting", + kind: "invalid" as const, + detail: this.#autoroutingEffective.issue.detail, + }, + ]), + ]; + return { + issues: structuredClone(issues), + valid: !issues.some(issue => issue.kind === "invalid"), + }; + } + + /** Effective merged autorouting state shared by settings diagnostics and routing policy. */ + getEffectiveAutorouting(): AutoroutingEffective { + return structuredClone(this.#autoroutingEffective); } onChanged(listener: (path: SettingPath) => void): () => void { @@ -1066,6 +1102,9 @@ export class Settings implements NotificationSettingsReader { }); cloned.#storage = this.#storage; cloned.#schemaReport = structuredClone(this.#schemaReport); + + cloned.#autoroutingEffective = structuredClone(this.#autoroutingEffective); + cloned.#autoroutingLocalIssues = structuredClone(this.#autoroutingLocalIssues); cloned.#schemaMigrationPending = this.#schemaMigrationPending; cloned.#futureSchemaVersion = this.#futureSchemaVersion; cloned.#hasMalformedConfigRoot = this.#hasMalformedConfigRoot; @@ -1601,6 +1640,15 @@ export class Settings implements NotificationSettingsReader { } setByPath(source, pathSegments, sanitized); } + + const tiersPath = ["task", "autorouting", "tiers"]; + const tiers = getByPath(source, tiersPath); + if (tiers === undefined) continue; + if (!tiers || typeof tiers !== "object" || Array.isArray(tiers)) { + logger.warn("Settings: retained malformed autorouting tier record for schema diagnostics", { + path: tiersPath.join("."), + }); + } } } @@ -5860,6 +5908,18 @@ export class Settings implements NotificationSettingsReader { throw error; } } + #recomputeAutoroutingDiagnostic(): void { + this.#autoroutingLocalIssues = [this.#global, this.#project, this.#overrides].flatMap(source => { + const fragment = getByPath(source, ["task", "autorouting"]); + return validateAutoroutingLocal(fragment).map(localIssue => ({ + path: localIssue.path ? `task.autorouting.${localIssue.path}` : "task.autorouting", + kind: "invalid" as const, + detail: localIssue.detail, + })); + }); + this.#autoroutingEffective = validateAutoroutingEffective(getByPath(this.#merged, ["task", "autorouting"])); + } + #rebuildMerged(): void { const project = structuredClone(this.#project); const overrides = structuredClone(this.#overrides); @@ -5870,6 +5930,7 @@ export class Settings implements NotificationSettingsReader { } this.#merged = this.#deepMerge(this.#deepMerge({}, this.#global), project); this.#merged = this.#deepMerge(this.#merged, overrides); + this.#recomputeAutoroutingDiagnostic(); } #fireAllHooks(): void { diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index f789349fe0..b5c3454307 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -173,6 +173,8 @@ const ACP_DEFAULTED_SETTING_PATHS: SettingPath[] = [ "task.maxRecursionDepth", "task.disabledAgents", "task.agentModelOverrides", + "task.autorouting.enabled", + "task.autorouting.tiers", // Memory subsystems are off-by-default for embedded (ACP) hosts; embedders // that want memory should opt in explicitly through their own settings layer. "memory.backend", diff --git a/packages/coding-agent/src/modes/acp/acp-agent.ts b/packages/coding-agent/src/modes/acp/acp-agent.ts index 3e52bd94f6..5286e123df 100644 --- a/packages/coding-agent/src/modes/acp/acp-agent.ts +++ b/packages/coding-agent/src/modes/acp/acp-agent.ts @@ -165,6 +165,8 @@ type SessionRecord = { /** Bounded set of correlations already settled; they stay closed for publication. */ settledPromptCorrelations: PromptCorrelation[]; authFailure?: string; + /** Replayable startup notice captured before ACP bootstrap; emitted once after the session id is known. */ + routingInactiveNotice?: string; activePrompt?: PromptWaiter; /** Set by `session/cancel` so an in-flight prompt settles as `cancelled`, never as an error. */ cancelRequested?: boolean; @@ -2626,6 +2628,10 @@ export class AcpAgent implements Agent { if (!received) return; const { event, wirePayload } = received; const isTerminal = event.type === "agent_end" || event.type === "agent_failed"; + if (event.type === "notice" && event.source === "autorouting" && typeof event.message === "string") { + record.routingInactiveNotice = event.message; + return; + } const derivedCorrelation = sdkFrameCorrelation(frame, event); const correlation = derivedCorrelation ?? {}; const activePrompt = record.activePrompt; @@ -3441,6 +3447,24 @@ export class AcpAgent implements Agent { record.adapter, ); } + // Not consumed on publish: this mirrors authFailure's lifecycle, where a + // later load/resume legitimately re-announces the condition. Clearing here + // would also lose the warning outright if the publish below rejected, since + // the enclosing bootstrap task swallows failures. + if (record.routingInactiveNotice) { + const message = record.routingInactiveNotice; + await this.#publishSessionUpdate( + id, + { + sessionId: id, + update: { + sessionUpdate: "agent_thought_chunk", + content: { type: "text", text: `[warning:autorouting] ${message}\n` }, + }, + }, + record.adapter, + ); + } await this.#publishSessionUpdate( id, { diff --git a/packages/coding-agent/src/modes/components/model-selector.ts b/packages/coding-agent/src/modes/components/model-selector.ts index 08e18c4b9d..7de90f32d0 100644 --- a/packages/coding-agent/src/modes/components/model-selector.ts +++ b/packages/coding-agent/src/modes/components/model-selector.ts @@ -14,6 +14,15 @@ import { truncateToWidth, } from "@gajae-code/tui"; import { sanitizeText } from "@gajae-code/utils"; +import { + type AutoroutingProviderOrderHint, + type AutoroutingSetup, + autoroutingProviderOrderHint, + evaluateAutoroutingProvenanceState, + normalizeTierMap, + validateAutoroutingSetup, +} from "../../config/autorouting-contract"; + import { isModelProfileProviderAvailable } from "../../config/model-profile-contract"; import { getModelProfilePresentation, @@ -46,6 +55,8 @@ import { formatModelOnboardingInlineHint } from "../../setup/model-onboarding-gu import { formatClampedModelSelector, getThinkingLevelMetadata, parseThinkingLevel } from "../../thinking"; import { getTabBarTheme } from "../shared"; import { DynamicBorder } from "./dynamic-border"; +import type { SmartRoutingIntent, SmartRoutingPreview } from "./smart-routing-panel"; +import { SmartRoutingPanelComponent } from "./smart-routing-panel"; function makeInvertedBadge(label: string, color: ThemeColor): string { const fgAnsi = theme.getFgAnsi(color); @@ -147,6 +158,10 @@ export type ModelSelectorSelection = | { kind: "deleteProfile"; profileName: string; + } + | { + kind: "smartRouting"; + intent: SmartRoutingIntent; }; interface PendingThinkingChoice { @@ -180,7 +195,7 @@ function createProviderTab(providerId: string): ProviderTabState { return { id: providerId, label: formatProviderTabLabel(providerId), providerId }; } -type ModelSelectorViewMode = "presets" | "models"; +type ModelSelectorViewMode = "presets" | "models" | "smart-routing"; interface PresetGroupRow { kind: "group"; @@ -216,6 +231,10 @@ interface PresetImageRoleRow { kind: "imageRole"; } +interface PresetSmartRoutingRow { + kind: "smartRouting"; +} + type PresetLandingRow = | PresetGroupRow | PresetProfileRow @@ -223,7 +242,8 @@ type PresetLandingRow = | PresetCreateUnavailableRow | PresetAlreadySavedRow | PresetBrowseRow - | PresetImageRoleRow; + | PresetImageRoleRow + | PresetSmartRoutingRow; // Stable logical identity for a preset landing row, independent of its current // list position. Used to relocate the cursor after the expanded group changes so @@ -244,6 +264,8 @@ function presetRowIdentity(row: PresetLandingRow): string { return `alreadySaved:${row.profile.name}`; case "imageRole": return "imageRole"; + case "smartRouting": + return "smartRouting"; } } @@ -376,7 +398,10 @@ export class ModelSelectorComponent extends Container { #assignmentState: "idle" | "assigning" = "idle"; #closeAfterAssignment = false; #unsubscribeCatalogChanged: () => void = () => {}; + #unsubscribeProviderOrderChanged: () => void = () => {}; #disposed = false; + /** Standalone smart-routing entry: cancel closes the selector instead of falling back to the preset landing. */ + #smartRoutingOnly = false; // Preset landing state #viewMode: ModelSelectorViewMode = "presets"; @@ -391,6 +416,8 @@ export class ModelSelectorComponent extends Container { #presetLoginHint?: string; #authSessionId?: string; #imageRoleFilter: boolean = false; + #smartRoutingPanel?: SmartRoutingPanelComponent; + #smartRoutingPreviewBuilder?: (draft: AutoroutingSetup) => SmartRoutingPreview; // Tab state #providers: ProviderTabState[] = STATIC_PROVIDER_TABS; @@ -414,6 +441,9 @@ export class ModelSelectorComponent extends Container { currentThinkingLevel?: ThinkingLevel; activeModelProfile?: string; configuredDefaultChain?: readonly string[]; + smartRoutingPreview?: (draft: AutoroutingSetup) => SmartRoutingPreview; + /** Open the smart-routing panel directly instead of the preset landing. */ + smartRoutingOnly?: boolean; }, ) { super(); @@ -426,6 +456,7 @@ export class ModelSelectorComponent extends Container { this.#onCancelCallback = onCancel; this.#temporaryOnly = options?.temporaryOnly ?? false; this.#authSessionId = options?.sessionId; + this.#smartRoutingPreviewBuilder = options?.smartRoutingPreview; this.#currentModel = _currentModel; this.#currentThinkingLevel = options?.currentThinkingLevel; this.#activeModelProfile = options?.activeModelProfile; @@ -442,7 +473,12 @@ export class ModelSelectorComponent extends Container { ? this.#isFastForProvider(this.#currentModel.provider, modelSupportsServiceTier(this.#currentModel)) : false); const initialSearchInput = options?.initialSearchInput; - this.#viewMode = this.#temporaryOnly || initialSearchInput || scopedModels.length > 0 ? "models" : "presets"; + this.#smartRoutingOnly = options?.smartRoutingOnly === true; + this.#viewMode = this.#smartRoutingOnly + ? "smart-routing" + : this.#temporaryOnly || initialSearchInput || scopedModels.length > 0 + ? "models" + : "presets"; // Load current role assignments from settings this.#rebuildRoleModels(); @@ -492,9 +528,25 @@ export class ModelSelectorComponent extends Container { }); } + // Advisory drift only, and only while the smart-routing panel is mounted. This + // must not call refreshState: that would discard the user's unsaved draft. + this.#unsubscribeProviderOrderChanged = this.#settings.onChanged?.(path => { + if (this.#disposed || path !== "modelProviderOrder") return; + if (this.#viewMode !== "smart-routing") return; + const panel = this.#smartRoutingPanel; + if (!panel) return; + panel.updateProviderOrderHint(this.#providerOrderHintFor(panel.getProviderOrder())); + this.#tui.requestRender(); + }); + // Load models and do initial render this.#loadModels().then(() => { this.#buildProviderTabs(); + if (this.#smartRoutingOnly) { + this.#enterSmartRoutingMode(); + this.#tui.requestRender(); + return; + } if (this.#viewMode === "presets" && (this.#modelRegistry.getModelProfiles?.().size ?? 0) === 0) { this.#viewMode = "models"; } @@ -521,6 +573,7 @@ export class ModelSelectorComponent extends Container { if (this.#disposed) return; this.#disposed = true; this.#unsubscribeCatalogChanged(); + this.#unsubscribeProviderOrderChanged(); super.dispose(); } @@ -1230,6 +1283,7 @@ export class ModelSelectorComponent extends Container { rows.push({ kind: "createUnavailable", label: "Select a model before creating a custom preset" }); } rows.push({ kind: "imageRole" }); + rows.push({ kind: "smartRouting" }); rows.push({ kind: "browse" }); return rows; } @@ -1436,7 +1490,8 @@ export class ModelSelectorComponent extends Container { selected.kind === "create" || selected.kind === "createUnavailable" || selected.kind === "alreadySaved" || - selected.kind === "imageRole" + selected.kind === "imageRole" || + selected.kind === "smartRouting" ) return; if (this.#expandedPresetProviderId === selected.groupId) return; @@ -1453,7 +1508,8 @@ export class ModelSelectorComponent extends Container { selected.kind === "create" || selected.kind === "createUnavailable" || selected.kind === "alreadySaved" || - selected.kind === "imageRole" + selected.kind === "imageRole" || + selected.kind === "smartRouting" ) return; if (this.#expandedPresetProviderId !== selected.groupId) return; @@ -1519,6 +1575,128 @@ export class ModelSelectorComponent extends Container { return truncateToWidth(label, ROLE_BINDING_MAX_WIDTH); } + /** + * A recorded declaration always wins; otherwise seed from the deterministic + * provider priority so the draft reflects the user's configured order instead of + * raw catalog iteration. No hardcoded provider fallback: an empty catalog means + * there is nothing to generate tiers from, and the caller refuses entry. + */ + #smartRoutingSetup(): AutoroutingSetup { + const stored = this.#settings.get("task.autorouting.setup"); + if (validateAutoroutingSetup(stored).length === 0 && stored !== undefined) return structuredClone(stored); + return { schema: 1, providers: [...this.#modelRegistry.autoroutingProviderOrder()] }; + } + + /** + * Advisory hint input is the panel's *current draft*, not the persisted setup, so + * a user mid-reorder sees drift for what they are actually editing. + */ + #providerOrderHintFor(declared: readonly string[]): AutoroutingProviderOrderHint { + return autoroutingProviderOrderHint(declared, this.#modelRegistry.autoroutingProviderOrder()); + } + + #smartRoutingPreview(setup: AutoroutingSetup): SmartRoutingPreview { + if (!this.#smartRoutingPreviewBuilder) { + throw new Error("Smart-routing preview is unavailable in this selector context."); + } + return this.#smartRoutingPreviewBuilder(setup); + } + + #smartRoutingIsStale(): boolean { + const provenance = this.#settings.get("task.autorouting.provenance"); + if (!provenance || validateAutoroutingSetup(this.#smartRoutingSetup()).length > 0) return false; + try { + const preview = this.#smartRoutingPreview(this.#smartRoutingSetup()); + const state = evaluateAutoroutingProvenanceState(provenance, { + catalogFingerprint: preview.sourceIdentity.catalogFingerprint, + mapFingerprint: preview.sourceIdentity.mapFingerprint, + tiers: this.#settings.get("task.autorouting.tiers") ?? {}, + }); + return state.staleMap || state.staleCatalog || state.handEdited; + } catch { + return true; + } + } + + #smartRoutingReadOnly(): boolean { + return this.#temporaryOnly || this.#scopedModels.length > 0 || !this.#settings.canWriteDurableConfig(); + } + + #enterSmartRoutingMode(): void { + if (!this.#smartRoutingPreviewBuilder) { + if (this.#smartRoutingOnly) { + this.#onCancelCallback(); + return; + } + this.#presetLoginHint = "Smart-routing setup is unavailable in this selector context."; + this.#renderPresetLanding(); + return; + } + + const setup = this.#smartRoutingSetup(); + if (setup.providers.length === 0) { + // Nothing to generate tiers from; refuse entry instead of seeding a guess. + if (this.#smartRoutingOnly) { + this.#onCancelCallback(); + return; + } + this.#presetLoginHint = "No providers are available to generate routing tiers."; + this.#renderPresetLanding(); + return; + } + const preview = this.#smartRoutingPreview(setup); + const tiers = normalizeTierMap(this.#settings.get("task.autorouting.tiers")); + const provenance = this.#settings.get("task.autorouting.provenance"); + this.#viewMode = "smart-routing"; + this.#smartRoutingPanel = new SmartRoutingPanelComponent({ + setup, + tiers, + provenance, + enabled: this.#settings.get("task.autorouting.enabled") === true, + providerOrderHint: this.#providerOrderHintFor(setup.providers), + readOnly: this.#smartRoutingReadOnly(), + stale: this.#smartRoutingIsStale(), + preview, + generatePreview: draft => this.#smartRoutingPreview(draft), + onSelect: async intent => { + await this.#onSelectCallback({ kind: "smartRouting", intent }); + return intent.kind === "apply" ? this.#smartRoutingPreview(intent.draft) : undefined; + }, + onCancel: () => (this.#smartRoutingOnly ? this.#onCancelCallback() : this.#switchToPresetMode()), + }); + this.#headerContainer.clear(); + this.#headerContainer.addChild(new Text(theme.fg("accent", "Smart routing"), 0, 0)); + this.#tabBar = null; + this.#listContainer.clear(); + this.#listContainer.addChild(this.#smartRoutingPanel); + this.#tui.requestRender(); + } + + #switchToPresetMode(): void { + this.#smartRoutingPanel = undefined; + this.#viewMode = "presets"; + this.#presetCursor = Math.min(this.#presetCursor, Math.max(0, this.#getPresetRows().length - 1)); + this.#renderPresetLanding(); + this.#tui.requestRender(); + } + + refreshSmartRoutingState(): void { + const panel = this.#smartRoutingPanel; + if (!panel || this.#viewMode !== "smart-routing") return; + const setup = this.#smartRoutingSetup(); + const preview = this.#smartRoutingPreview(setup); + panel.refreshState({ + setup, + tiers: normalizeTierMap(this.#settings.get("task.autorouting.tiers")), + provenance: this.#settings.get("task.autorouting.provenance"), + enabled: this.#settings.get("task.autorouting.enabled") === true, + providerOrderHint: this.#providerOrderHintFor(setup.providers), + stale: this.#smartRoutingIsStale(), + preview, + }); + this.#tui.requestRender(); + } + #renderPresetLanding(): void { this.#headerContainer.clear(); this.#tabBar = null; @@ -1532,6 +1710,12 @@ export class ModelSelectorComponent extends Container { const row = rows[i]; const selected = i === this.#presetCursor; const prefix = selected ? theme.fg("accent", `${theme.nav.cursor} `) : " "; + if (row.kind === "smartRouting") { + const stale = this.#smartRoutingIsStale(); + const label = stale ? "Smart routing (stale)" : "Smart routing setup"; + this.#listContainer.addChild(new Text(`${prefix}${selected ? theme.fg("accent", label) : label}`, 0, 0)); + continue; + } if (row.kind === "create") { const label = "Create custom preset"; this.#listContainer.addChild(new Text(`${prefix}${selected ? theme.fg("accent", label) : label}`, 0, 0)); @@ -1922,6 +2106,10 @@ export class ModelSelectorComponent extends Container { } handleInput(keyData: string): void { + if (this.#viewMode === "smart-routing") { + this.#smartRoutingPanel?.handleInput(keyData); + return; + } if (this.#assignmentState === "assigning") { if (getKeybindings().matches(keyData, "tui.select.cancel")) { this.#closeAfterAssignment = true; @@ -2107,6 +2295,10 @@ export class ModelSelectorComponent extends Container { } const row = this.#getSelectedPresetRow(); if (!row) return; + if (row.kind === "smartRouting") { + this.#enterSmartRoutingMode(); + return; + } if (row.kind === "create") { this.#onSelectCallback({ kind: "createProfile", profile: this.#buildCustomModelProfileSnapshot() }); return; @@ -2432,6 +2624,16 @@ export class ModelSelectorComponent extends Container { const row = this.#getSelectedPresetRow(); return row ? presetRowIdentity(row) : undefined; } + __testGetSmartRoutingPanel(): SmartRoutingPanelComponent | undefined { + return this.#smartRoutingPanel; + } + + __testViewMode(): ModelSelectorViewMode { + return this.#viewMode; + } + __testOpenSmartRoutingPanel(): void { + this.#enterSmartRoutingMode(); + } } function getSelectableThinkingLevels(model: Model): ThinkingLevel[] { diff --git a/packages/coding-agent/src/modes/components/smart-routing-panel.ts b/packages/coding-agent/src/modes/components/smart-routing-panel.ts new file mode 100644 index 0000000000..02d3a6295a --- /dev/null +++ b/packages/coding-agent/src/modes/components/smart-routing-panel.ts @@ -0,0 +1,531 @@ +import { Container, getKeybindings, matchesKey, replaceTabs, Spacer, Text, truncateToWidth } from "@gajae-code/tui"; +import { sanitizeDisplayLine } from "@gajae-code/utils"; +import type { + AutoroutingProvenance, + AutoroutingProviderOrderHint, + AutoroutingSetup, + AutoroutingTier, + TierMap, +} from "../../config/autorouting-contract"; +import type { AutoroutingSourceIdentity } from "../../config/autorouting-generator"; +import { theme } from "../theme/theme"; + +/** Longest rendered panel line before truncation. */ +export const MAX_PANEL_LINE_WIDTH = 200; + +export type SmartRoutingPreview = { + readonly setup: AutoroutingSetup; + readonly tiers: TierMap; + readonly provenance: AutoroutingProvenance; + readonly sourceIdentity: AutoroutingSourceIdentity; +}; + +export type SmartRoutingIntent = + | { + kind: "apply"; + draft: AutoroutingSetup; + preview: SmartRoutingPreview; + confirmHandEdit?: boolean; + } + | { + kind: "refresh"; + confirmHandEdit?: boolean; + } + | { + kind: "clear"; + } + | { + kind: "toggle"; + enabled: boolean; + }; + +export type SmartRoutingPanelMode = "editing" | "confirming" | "committing" | "done" | "error"; +export type SmartRoutingConfirmation = "clear" | "hand-edit" | undefined; + +export interface SmartRoutingPanelOptions { + setup: AutoroutingSetup; + tiers?: TierMap; + provenance?: AutoroutingProvenance; + /** Advisory drift between the recorded declaration and current provider priority. */ + providerOrderHint?: AutoroutingProviderOrderHint; + enabled: boolean; + readOnly: boolean; + stale: boolean; + preview: SmartRoutingPreview; + generatePreview: (draft: AutoroutingSetup) => SmartRoutingPreview; + onSelect: (intent: SmartRoutingIntent) => undefined | Promise; + onCancel: () => void; +} + +function cloneSetup(setup: AutoroutingSetup): AutoroutingSetup { + return structuredClone(setup); +} + +function clonePreview(preview: SmartRoutingPreview): SmartRoutingPreview { + return structuredClone(preview); +} + +function isPrintable(data: string): boolean { + return data.length === 1 && data >= " " && data !== "\x7f"; +} + +function isBackspace(data: string): boolean { + return data === "\x7f" || data === "\b"; +} + +/** + * Provider names, allowlist entries, generated selectors, and error text all + * originate in hand-editable config or catalog data, so any of them can carry + * tabs, control bytes, line breaks, or terminal escape sequences. Any of those + * can inject extra rows or evade the width cap, so flatten and bound the value + * before it reaches a renderer. + */ +function displaySafe(text: string): string { + return truncateToWidth(replaceTabs(sanitizeDisplayLine(text)), MAX_PANEL_LINE_WIDTH); +} + +function formatTier(tier: AutoroutingTier, tiers: TierMap | undefined): string { + const selectors = tiers?.[tier]; + return displaySafe( + `${tier}: ${selectors && selectors.length > 0 ? selectors.join(", ") : "(empty; manual fallback)"}`, + ); +} + +/** + * Presentational smart-routing setup editor. It owns only ephemeral draft and + * preview state; all durable changes are emitted as typed intents. + */ +export class SmartRoutingPanelComponent extends Container { + readonly #onSelect: SmartRoutingPanelOptions["onSelect"]; + readonly #onCancel: () => void; + readonly #generatePreview: (draft: AutoroutingSetup) => SmartRoutingPreview; + readonly #readOnly: boolean; + #draft: AutoroutingSetup; + + #tiers?: TierMap; + #provenance?: AutoroutingProvenance; + #enabled: boolean; + #stale: boolean; + #preview: SmartRoutingPreview; + #mode: SmartRoutingPanelMode = "editing"; + #confirmation: SmartRoutingConfirmation; + /** Intent that triggered the current hand-edit confirmation, replayed verbatim on confirm. */ + #pendingIntent: SmartRoutingIntent | undefined; + #providerCursor = 0; + #allowlistEditing = false; + #allowlistBuffer = ""; + #providerOrderHint: AutoroutingProviderOrderHint | undefined; + #status = ""; + #error = ""; + + constructor(options: SmartRoutingPanelOptions) { + super(); + this.#onSelect = options.onSelect; + this.#onCancel = options.onCancel; + this.#generatePreview = options.generatePreview; + this.#readOnly = options.readOnly; + this.#draft = cloneSetup(options.setup); + + this.#tiers = options.tiers ? structuredClone(options.tiers) : undefined; + this.#provenance = options.provenance ? structuredClone(options.provenance) : undefined; + this.#enabled = options.enabled; + this.#providerOrderHint = options.providerOrderHint ? structuredClone(options.providerOrderHint) : undefined; + this.#stale = options.stale; + this.#preview = clonePreview(options.preview); + this.#render(); + } + + get mode(): SmartRoutingPanelMode { + return this.#mode; + } + + get confirmation(): SmartRoutingConfirmation { + return this.#confirmation; + } + + getDraft(): AutoroutingSetup { + return cloneSetup(this.#draft); + } + + getPreviewPayload(): SmartRoutingPreview { + return clonePreview(this.#preview); + } + + getProviderOrder(): readonly string[] { + return [...this.#draft.providers]; + } + + /** + * Advisory-only, non-destructive hint update. + * + * `refreshState` deliberately replaces the draft and clears editor state, so it + * must never be used for a background settings change: an external + * `modelProviderOrder` edit would then discard the user's unsaved reordering, + * allowlist buffer, cursor, and confirmation. This replaces the hint and nothing + * else. + */ + updateProviderOrderHint(hint: AutoroutingProviderOrderHint | undefined): void { + this.#providerOrderHint = hint ? structuredClone(hint) : undefined; + this.#render(); + } + + /** Replace the panel's persisted snapshot after a controller mutation. */ + refreshState(options: { + setup: AutoroutingSetup; + tiers?: TierMap; + provenance?: AutoroutingProvenance; + enabled: boolean; + providerOrderHint?: AutoroutingProviderOrderHint; + stale: boolean; + preview: SmartRoutingPreview; + }): void { + this.#draft = cloneSetup(options.setup); + this.#tiers = options.tiers ? structuredClone(options.tiers) : undefined; + this.#provenance = options.provenance ? structuredClone(options.provenance) : undefined; + this.#enabled = options.enabled; + this.#stale = options.stale; + this.#providerOrderHint = options.providerOrderHint ? structuredClone(options.providerOrderHint) : undefined; + this.#preview = clonePreview(options.preview); + this.#providerCursor = Math.min(this.#providerCursor, Math.max(0, this.#draft.providers.length - 1)); + this.#mode = "editing"; + this.#confirmation = undefined; + this.#status = ""; + this.#error = ""; + this.#render(); + } + + #render(): void { + this.detachAll(); + this.addChild(new Text(theme.fg("accent", "Smart routing setup"), 0, 0)); + this.addChild( + new Text(theme.fg("muted", "Declare providers in priority order; generated chains are deterministic."), 0, 0), + ); + if (this.#readOnly) { + this.addChild( + new Text( + theme.fg("warning", "Read-only: temporary or --models-scoped sessions cannot write autorouting."), + 0, + 0, + ), + ); + } + if (this.#stale) { + this.addChild( + new Text( + theme.fg("warning", "Stale generated setup: catalog/map or hand-edited tiers differ from provenance."), + 0, + 0, + ), + ); + } + // Advisory only: a changed provider priority is a new suggestion, never proof + // that the persisted tiers went stale. + if (this.#providerOrderHint?.reordered) { + this.addChild( + new Text( + theme.fg("muted", "Provider priority changed since this setup was generated. Press r to refresh."), + 0, + 0, + ), + ); + } + if (this.#providerOrderHint && this.#providerOrderHint.missing.length > 0) { + this.addChild( + new Text( + theme.fg( + "muted", + displaySafe( + `Declared providers missing from the catalog: ${this.#providerOrderHint.missing.join(", ")}`, + ), + ), + 0, + 0, + ), + ); + } + if (this.#provenance) + this.addChild(new Text(theme.fg("dim", "Provenance recorded for this generated setup."), 0, 0)); + this.addChild(new Spacer(1)); + this.addChild(new Text(theme.fg("muted", `Enabled: ${this.#enabled ? "yes" : "no"}`), 0, 0)); + this.addChild(new Text(theme.fg("muted", "Providers (↑/↓ reorder; x removes; m edits allowlist):"), 0, 0)); + for (let index = 0; index < this.#draft.providers.length; index++) { + const provider = this.#draft.providers[index] ?? ""; + const prefix = index === this.#providerCursor ? theme.fg("accent", `${theme.nav.cursor} `) : " "; + this.addChild(new Text(`${prefix}${displaySafe(provider)}`, 0, 0)); + } + if (this.#draft.providers.length === 0) + this.addChild(new Text(theme.fg("error", " No providers declared."), 0, 0)); + if (this.#allowlistEditing) { + this.addChild( + new Text(theme.fg("accent", displaySafe(`Allowlist: ${this.#allowlistBuffer || "(all models)"}`)), 0, 0), + ); + this.addChild( + new Text(theme.fg("muted", " Type provider/model values separated by commas, then Enter."), 0, 0), + ); + } else { + this.addChild( + new Text( + theme.fg( + "dim", + displaySafe( + `Allowlist: ${this.#draft.models && this.#draft.models.length > 0 ? this.#draft.models.join(", ") : "(all labeled models)"}`, + ), + ), + 0, + 0, + ), + ); + } + this.addChild(new Spacer(1)); + this.addChild(new Text(theme.fg("muted", "Preview:"), 0, 0)); + for (const tier of ["fast", "balanced", "strong"] as const) { + this.addChild(new Text(` ${formatTier(tier, this.#preview.tiers)}`, 0, 0)); + } + if (this.#status) this.addChild(new Text(theme.fg("success", displaySafe(this.#status)), 0, 0)); + if (this.#error) this.addChild(new Text(theme.fg("error", displaySafe(this.#error)), 0, 0)); + if (this.#confirmation === "clear") { + this.addChild( + new Text( + theme.fg("warning", "Clear generated tiers, setup, and provenance? Enter confirms; Esc cancels."), + 0, + 0, + ), + ); + } else if (this.#confirmation === "hand-edit") { + this.addChild( + new Text( + theme.fg( + "warning", + "Stored tiers were hand-edited. Regenerate and replace them? Enter confirms; Esc cancels.", + ), + 0, + 0, + ), + ); + } + this.addChild(new Spacer(1)); + const actions = this.#readOnly ? "Esc: cancel" : "a: apply · r: refresh · t: toggle · c: clear · Esc: cancel"; + this.addChild(new Text(theme.fg("muted", actions), 0, 0)); + } + + #recomputePreview(): void { + try { + this.#preview = clonePreview(this.#generatePreview(this.#draft)); + this.#error = ""; + } catch (error) { + this.#error = error instanceof Error ? error.message : String(error); + } + } + + async #emit(intent: SmartRoutingIntent): Promise { + if (this.#readOnly || this.#mode === "committing") return; + this.#mode = "committing"; + this.#status = "Committing smart-routing settings…"; + this.#error = ""; + this.#confirmation = undefined; + this.#render(); + try { + const result = await this.#onSelect(intent); + if (intent.kind === "apply") { + const appliedPreview = result ?? this.#generatePreview(intent.draft); + this.#draft = cloneSetup(appliedPreview.setup); + this.#preview = clonePreview(appliedPreview); + } else if (intent.kind === "toggle") { + this.#enabled = intent.enabled; + } + this.#mode = "done"; + this.#status = intent.kind === "clear" ? "Generated setup cleared." : "Smart-routing settings saved."; + } catch (error) { + const typed = error as { code?: unknown } | undefined; + if (typed?.code === "autorouting-hand-edited") { + this.#mode = "confirming"; + this.#confirmation = "hand-edit"; + // Retain the intent that triggered the guard so confirming replays THAT intent. An Apply + // carrying an edited draft must not be silently downgraded to a Refresh, which would + // discard the frozen preview and commit the previously recorded setup instead (AC6). + this.#pendingIntent = intent; + this.#status = ""; + this.#error = ""; + } else { + this.#mode = "error"; + this.#error = error instanceof Error ? error.message : String(error); + this.#status = ""; + } + } + this.#render(); + } + + #beginClearConfirmation(): void { + if (this.#readOnly || this.#mode !== "editing") return; + this.#mode = "confirming"; + this.#confirmation = "clear"; + this.#render(); + } + + #commitAllowlistBuffer(): void { + const values = this.#allowlistBuffer + .split(",") + .map(value => value.trim()) + .filter(value => value.length > 0); + this.#draft = + values.length > 0 + ? { ...this.#draft, models: [...new Set(values)] } + : { schema: 1, providers: [...this.#draft.providers] }; + this.#allowlistEditing = false; + this.#allowlistBuffer = ""; + this.#recomputePreview(); + this.#render(); + } + + handleInput(data: string): void { + if (this.#mode === "committing") return; + if (this.#mode === "done" || this.#mode === "error") { + if (getKeybindings().matches(data, "tui.select.cancel")) this.#onCancel(); + return; + } + if (this.#confirmation) { + if (getKeybindings().matches(data, "tui.select.cancel")) { + this.#confirmation = undefined; + this.#pendingIntent = undefined; + this.#mode = "editing"; + this.#render(); + return; + } + if (matchesKey(data, "enter") || matchesKey(data, "return") || data === "\n") { + void this.#emitConfirmed(); + } + return; + } + if (this.#allowlistEditing) { + if (matchesKey(data, "enter") || matchesKey(data, "return") || data === "\n") { + this.#commitAllowlistBuffer(); + return; + } + if (getKeybindings().matches(data, "tui.select.cancel")) { + this.#allowlistEditing = false; + this.#allowlistBuffer = ""; + this.#render(); + return; + } + if (isBackspace(data)) { + this.#allowlistBuffer = this.#allowlistBuffer.slice(0, -1); + this.#render(); + return; + } + if (isPrintable(data)) { + this.#allowlistBuffer += data; + this.#render(); + } + return; + } + if (getKeybindings().matches(data, "tui.select.cancel")) { + this.#onCancel(); + return; + } + if (this.#readOnly) return; + if (data === "m" || data === "M") { + this.#allowlistEditing = true; + this.#allowlistBuffer = this.#draft.models?.join(", ") ?? ""; + this.#render(); + return; + } + if (data === "x" || data === "X") { + if (this.#draft.providers.length <= 1) { + this.#error = "At least one provider must remain declared."; + this.#render(); + return; + } + const providers = this.#draft.providers.filter((_, index) => index !== this.#providerCursor); + this.#draft = { ...this.#draft, providers }; + this.#providerCursor = Math.min(this.#providerCursor, providers.length - 1); + this.#recomputePreview(); + this.#render(); + return; + } + if (matchesKey(data, "up")) { + if (this.#providerCursor > 0) { + const providers = [...this.#draft.providers]; + [providers[this.#providerCursor - 1], providers[this.#providerCursor]] = [ + providers[this.#providerCursor]!, + providers[this.#providerCursor - 1]!, + ]; + this.#draft = { ...this.#draft, providers }; + this.#providerCursor--; + this.#recomputePreview(); + this.#render(); + } + return; + } + if (matchesKey(data, "down")) { + if (this.#providerCursor < this.#draft.providers.length - 1) { + const providers = [...this.#draft.providers]; + [providers[this.#providerCursor], providers[this.#providerCursor + 1]] = [ + providers[this.#providerCursor + 1]!, + providers[this.#providerCursor]!, + ]; + this.#draft = { ...this.#draft, providers }; + this.#providerCursor++; + this.#recomputePreview(); + this.#render(); + } + return; + } + if (data === "a" || data === "A") { + void this.#emit({ kind: "apply", draft: cloneSetup(this.#draft), preview: clonePreview(this.#preview) }); + return; + } + if (data === "r" || data === "R") { + void this.#emit({ kind: "refresh" }); + return; + } + if (data === "t" || data === "T") { + void this.#emit({ kind: "toggle", enabled: !this.#enabled }); + return; + } + if (data === "c" || data === "C") { + this.#beginClearConfirmation(); + } + } + + __testApply(): Promise { + return this.#emit({ kind: "apply", draft: cloneSetup(this.#draft), preview: clonePreview(this.#preview) }); + } + + /** Test hook mirroring an in-panel provider reorder/edit: mutates the draft and re-previews. */ + __testSetProviders(providers: string[]): void { + this.#draft = { ...this.#draft, providers: [...providers] }; + this.#preview = this.#generatePreview(this.#draft); + this.#providerCursor = 0; + } + + __testRefresh(confirmHandEdit = false): Promise { + return this.#emit({ kind: "refresh", ...(confirmHandEdit ? { confirmHandEdit: true } : {}) }); + } + + __testToggle(enabled = !this.#enabled): Promise { + return this.#emit({ kind: "toggle", enabled }); + } + + __testClear(): Promise { + return this.#emit({ kind: "clear" }); + } + + __testConfirm(): Promise { + return this.#emitConfirmed(); + } + + /** + * Replay the intent the confirmation was raised for. A `clear` confirmation always emits `clear`; + * a hand-edit confirmation re-emits the ORIGINAL intent with `confirmHandEdit`, so a guarded Apply + * commits its own edited draft instead of being downgraded to a Refresh of the recorded setup. + */ + #emitConfirmed(): Promise { + if (this.#confirmation === "clear") return this.#emit({ kind: "clear" }); + if (this.#confirmation !== "hand-edit") return Promise.resolve(); + const pending = this.#pendingIntent; + this.#pendingIntent = undefined; + if (pending?.kind === "apply") + return this.#emit({ kind: "apply", draft: pending.draft, preview: pending.preview, confirmHandEdit: true }); + return this.#emit({ kind: "refresh", confirmHandEdit: true }); + } +} diff --git a/packages/coding-agent/src/modes/controllers/selector-controller.ts b/packages/coding-agent/src/modes/controllers/selector-controller.ts index 29d465ddf0..c8b43b6c9f 100644 --- a/packages/coding-agent/src/modes/controllers/selector-controller.ts +++ b/packages/coding-agent/src/modes/controllers/selector-controller.ts @@ -12,6 +12,16 @@ import type { OAuthProvider } from "@gajae-code/ai/utils/oauth/types"; import type { Component, OverlayHandle, SlashCommand } from "@gajae-code/tui"; import { Input, Loader, resolvePetMode, Spacer, Text } from "@gajae-code/tui"; import { getAgentDbPath, getProjectDir, logger, VERSION } from "@gajae-code/utils"; +import { + type AutoroutingProvenance, + type AutoroutingSetup, + buildAutoroutingEnabledPatch, + buildAutoroutingSettingsBatch, + evaluateAutoroutingProvenanceState, + validateAutoroutingSetup, +} from "../../config/autorouting-contract"; +import { canonicalJsonBytes, generateTierChains } from "../../config/autorouting-generator"; +import { CURATED_TIER_MAP } from "../../config/autorouting-tier-map"; import type { AppKeybinding } from "../../config/keybindings"; import { activateModelProfile, @@ -167,7 +177,6 @@ import { PlanPreviewOverlay, type PlanPreviewResult, } from "../components/plan-preview-overlay"; - import { PluginSelectorComponent } from "../components/plugin-selector"; import { type ProviderOnboardingAction, @@ -179,6 +188,7 @@ import { SessionObserverOverlayComponent } from "../components/session-observer- import { SessionSelectorComponent } from "../components/session-selector"; import { dashboardSessions, SessionsDashboardComponent } from "../components/sessions-dashboard"; import { SettingsSelectorComponent } from "../components/settings-selector"; +import type { SmartRoutingPreview } from "../components/smart-routing-panel"; import { TasksPaneComponent } from "../components/tasks-pane"; import { ThemeSelectorComponent } from "../components/theme-selector"; import { ThinkingSelectorComponent } from "../components/thinking-selector"; @@ -1176,7 +1186,18 @@ interface DefaultAssignmentRollbackSnapshot { resumeDefaultSelector: string | undefined; fallbackRuntimeState: DefaultFallbackRuntimeState; } + +function sameCanonicalAutoroutingValue(left: unknown, right: unknown): boolean { + const leftBytes = canonicalJsonBytes(left); + const rightBytes = canonicalJsonBytes(right); + if (leftBytes.length !== rightBytes.length) return false; + for (let index = 0; index < leftBytes.length; index++) { + if (leftBytes[index] !== rightBytes[index]) return false; + } + return true; +} export class SelectorController { + #smartRoutingInFlight?: Promise; #transcriptViewerOpen = false; #transcriptViewer?: TranscriptViewerOverlay; #sessionsDashboardOpen = false; @@ -2274,7 +2295,190 @@ export class SelectorController { this.ctx.showStatus(persistDefault ? `Default model profile: ${profileLabel}` : `Model profile: ${profileLabel}`); } - showModelSelector(options?: { temporaryOnly?: boolean }): void { + /** Generate the immutable preview used by the smart-routing panel. */ + previewSmartRouting(draft: AutoroutingSetup): SmartRoutingPreview { + const issues = validateAutoroutingSetup(draft); + if (issues.length > 0) + throw new Error(issues.map(issue => `${issue.path || "setup"}: ${issue.detail}`).join("; ")); + const setup = structuredClone(draft); + const generated = generateTierChains(setup, CURATED_TIER_MAP, [...this.ctx.session.modelRegistry.getAll()]); + const provenance: AutoroutingProvenance = { + schema: 1, + source: structuredClone(generated.sourceIdentity), + declarationFingerprint: generated.declarationFingerprint, + tiersFingerprint: generated.tiersFingerprint, + }; + return { + setup, + tiers: structuredClone(generated.tiers), + provenance, + sourceIdentity: structuredClone(generated.sourceIdentity), + }; + } + + #assertSmartRoutingWritable(): void { + if ((this.ctx.session.scopedModels?.length ?? 0) > 0) { + throw new Error("Smart-routing settings are read-only in a --models-scoped session."); + } + if (!this.ctx.settings.canWriteDurableConfig()) { + throw new Error("Cannot change smart-routing settings while durable config is unavailable."); + } + } + + #assertSmartRoutingNotHandEdited(preview: SmartRoutingPreview, allowHandEdit: boolean): void { + if (allowHandEdit) return; + const provenance = this.ctx.settings.get("task.autorouting.provenance"); + const currentTiers = this.ctx.settings.get("task.autorouting.tiers"); + if (!provenance || currentTiers === undefined) return; + const state = evaluateAutoroutingProvenanceState(provenance, { + catalogFingerprint: preview.sourceIdentity.catalogFingerprint, + mapFingerprint: preview.sourceIdentity.mapFingerprint, + tiers: currentTiers, + }); + if (!state.handEdited) return; + throw Object.assign( + new Error("Generated autorouting tiers were hand-edited; explicit confirmation is required."), + { code: "autorouting-hand-edited" }, + ); + } + + async #runSmartRoutingIntent(label: string, operation: () => Promise): Promise { + if (this.#smartRoutingInFlight) throw new Error("Another smart-routing operation is already in progress."); + const task = (async () => { + this.ctx.showStatus(`${label} smart-routing settings…`); + try { + const result = await operation(); + await this.ctx.notifyConfigChanged?.(); + this.ctx.showStatus(`${label} smart-routing settings saved.`); + this.ctx.ui.requestRender(); + return result; + } catch (error) { + this.ctx.showError(error instanceof Error ? error.message : String(error)); + this.ctx.ui.requestRender(); + throw error; + } finally { + this.#smartRoutingInFlight = undefined; + } + })(); + this.#smartRoutingInFlight = task; + return task; + } + #reportSmartRoutingValidationError(error: unknown): never { + this.ctx.showError(error instanceof Error ? error.message : String(error)); + this.ctx.ui.requestRender(); + throw error; + } + + async applySmartRouting( + draft: AutoroutingSetup, + options?: { preview?: SmartRoutingPreview; confirmHandEdit?: boolean }, + ): Promise { + let preview: SmartRoutingPreview; + try { + this.#assertSmartRoutingWritable(); + const issues = validateAutoroutingSetup(draft); + if (issues.length > 0) { + throw new Error(issues.map(issue => `${issue.path || "setup"}: ${issue.detail}`).join("; ")); + } + preview = options?.preview ?? this.previewSmartRouting(draft); + const regenerated = this.previewSmartRouting(draft); + if (!sameCanonicalAutoroutingValue(preview.setup, regenerated.setup)) { + throw new Error("Smart-routing preview does not match the draft being applied."); + } + if ( + !sameCanonicalAutoroutingValue(preview.tiers, regenerated.tiers) || + !sameCanonicalAutoroutingValue(preview.provenance, regenerated.provenance) + ) { + // Never persist caller-supplied tier/provenance bytes that diverge from + // the declaration. Continue with the fresh canonical payload so Apply + // remains exactly the generated preview for legitimate callers. + preview = regenerated; + } + this.#assertSmartRoutingNotHandEdited(preview, options?.confirmHandEdit === true); + } catch (error) { + return this.#reportSmartRoutingValidationError(error); + } + return this.#runSmartRoutingIntent("Apply", async () => { + await this.ctx.settings.commitAtomicBatchWithCurrent(() => + buildAutoroutingSettingsBatch({ + tiers: preview.tiers, + setup: preview.setup, + provenance: preview.provenance, + }), + ); + return preview; + }); + } + + /** + * Reorder a recorded declaration into current provider priority and drop entries + * the catalog no longer offers. Comparison uses normalized ids while the result + * keeps the catalog's spelling, because the generator matches provider prefixes + * case-sensitively. + */ + #reseedProvidersFromPolicy(setup: AutoroutingSetup): AutoroutingSetup { + const order = this.ctx.session.modelRegistry.autoroutingProviderOrder(); + const declared = new Set(setup.providers.map(provider => provider.trim().toLowerCase())); + const providers = order.filter(provider => declared.has(provider.trim().toLowerCase())); + return { ...structuredClone(setup), providers }; + } + + async refreshSmartRouting(options?: { confirmHandEdit?: boolean }): Promise { + let preview: SmartRoutingPreview; + try { + this.#assertSmartRoutingWritable(); + const setup = this.ctx.settings.get("task.autorouting.setup"); + const issues = validateAutoroutingSetup(setup); + if (issues.length > 0 || setup === undefined) { + throw new Error("Cannot refresh smart routing without a valid recorded setup."); + } + // Reseed the recorded declaration against the current provider priority: + // reorder to policy order and drop providers the catalog no longer offers. + // Refusing an empty result keeps a dead declaration from being persisted. + const reseeded = this.#reseedProvidersFromPolicy(setup); + if (reseeded.providers.length === 0) { + throw new Error("No declared providers remain in the catalog; smart routing was not updated."); + } + preview = this.previewSmartRouting(reseeded); + this.#assertSmartRoutingNotHandEdited(preview, options?.confirmHandEdit === true); + } catch (error) { + return this.#reportSmartRoutingValidationError(error); + } + return this.#runSmartRoutingIntent("Refresh", async () => { + await this.ctx.settings.commitAtomicBatchWithCurrent(() => + buildAutoroutingSettingsBatch({ + tiers: preview.tiers, + setup: preview.setup, + provenance: preview.provenance, + }), + ); + return preview; + }); + } + + async clearGeneratedSetup(): Promise { + try { + this.#assertSmartRoutingWritable(); + } catch (error) { + return this.#reportSmartRoutingValidationError(error); + } + return this.#runSmartRoutingIntent("Clear", async () => { + await this.ctx.settings.commitAtomicBatchWithCurrent(() => buildAutoroutingSettingsBatch({ clear: true })); + }); + } + + async setAutoroutingEnabled(enabled: boolean): Promise { + try { + this.#assertSmartRoutingWritable(); + } catch (error) { + return this.#reportSmartRoutingValidationError(error); + } + return this.#runSmartRoutingIntent("Toggle", async () => { + await this.ctx.settings.commitAtomicBatchWithCurrent(() => [buildAutoroutingEnabledPatch(enabled)]); + }); + } + + showModelSelector(options?: { temporaryOnly?: boolean; smartRoutingOnly?: boolean }): void { this.showSelector(done => { let modelSelector: ModelSelectorComponent; const refreshRoleAssignments = () => { @@ -2292,6 +2496,27 @@ export class SelectorController { this.ctx.session.modelRegistry, this.ctx.session.scopedModels, async selection => { + if (selection.kind === "smartRouting") { + switch (selection.intent.kind) { + case "apply": + await this.applySmartRouting(selection.intent.draft, { + preview: selection.intent.preview, + confirmHandEdit: selection.intent.confirmHandEdit, + }); + break; + case "refresh": + await this.refreshSmartRouting({ confirmHandEdit: selection.intent.confirmHandEdit }); + break; + case "clear": + await this.clearGeneratedSetup(); + break; + case "toggle": + await this.setAutoroutingEnabled(selection.intent.enabled); + break; + } + modelSelector.refreshSmartRoutingState(); + return; + } const isTrackedSingleAssignment = selection.kind === "assignment" && selection.role !== null && selection.roles === undefined; try { @@ -2525,6 +2750,8 @@ export class SelectorController { isFastForSubagentProvider: (provider, supportsServiceTier) => this.ctx.session.isFastForSubagentProvider(provider, supportsServiceTier), isCurrentModelFastModeActive: () => this.ctx.session.isFastModeActive(), + smartRoutingPreview: draft => this.previewSmartRouting(draft), + smartRoutingOnly: options?.smartRoutingOnly, }, ); return { component: modelSelector, focus: modelSelector }; diff --git a/packages/coding-agent/src/modes/interactive-mode.ts b/packages/coding-agent/src/modes/interactive-mode.ts index 2821a1c498..a1b19ecd5f 100644 --- a/packages/coding-agent/src/modes/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive-mode.ts @@ -2351,10 +2351,14 @@ export class InteractiveMode implements InteractiveModeContext { void this.#selectorController.showAgentsDashboard(); } - showModelSelector(options?: { temporaryOnly?: boolean }): void { + showModelSelector(options?: { temporaryOnly?: boolean; smartRoutingOnly?: boolean }): void { this.#selectorController.showModelSelector(options); } + setAutoroutingEnabled(enabled: boolean): Promise { + return this.#selectorController.setAutoroutingEnabled(enabled); + } + showEffortSelector(): void { this.#selectorController.showEffortSelector(); } diff --git a/packages/coding-agent/src/modes/types.ts b/packages/coding-agent/src/modes/types.ts index c3ff8404c5..ff72b1b31f 100644 --- a/packages/coding-agent/src/modes/types.ts +++ b/packages/coding-agent/src/modes/types.ts @@ -411,7 +411,8 @@ export interface InteractiveModeContext { showExtensionsDashboard(): void; showCustomizationDashboard(): void; showAgentsDashboard(): void; - showModelSelector(options?: { temporaryOnly?: boolean }): void; + showModelSelector(options?: { temporaryOnly?: boolean; smartRoutingOnly?: boolean }): void; + setAutoroutingEnabled(enabled: boolean): Promise; showEffortSelector(): void; showProviderOnboarding(): void; showPluginSelector(mode?: "install" | "uninstall"): void; diff --git a/packages/coding-agent/src/prompts/tools/task-summary.md b/packages/coding-agent/src/prompts/tools/task-summary.md index d1b8216100..094721e81f 100644 --- a/packages/coding-agent/src/prompts/tools/task-summary.md +++ b/packages/coding-agent/src/prompts/tools/task-summary.md @@ -5,6 +5,7 @@ {{status}} {{#if meta}}{{/if}} +{{#if routing}}{{/if}} {{synopsis}} diff --git a/packages/coding-agent/src/sdk/bus/index.ts b/packages/coding-agent/src/sdk/bus/index.ts index c716015f67..9042febda5 100644 --- a/packages/coding-agent/src/sdk/bus/index.ts +++ b/packages/coding-agent/src/sdk/bus/index.ts @@ -48,6 +48,7 @@ function sdkBusNatives(): NativeSdkBusBindings { type NotificationServer = NativeNotificationServer; import { $credentialEnv, logger, postmortem, VERSION } from "@gajae-code/utils"; + import { AsyncJobManager } from "../../async"; import { Settings, validateSettingPatch } from "../../config/settings"; import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext } from "../../extensibility/extensions"; @@ -100,6 +101,7 @@ import { } from "../host"; import { type AbortScope, type ControlSurface, dispatchControl, TypedControlError } from "../host/control"; import { BROKER_RUNTIME_CLOSE_CAPABILITY_FIELD } from "../host/control/runtime-gate"; +import { isAutoroutingInactive, markAutoroutingInactive } from "../host/internal-autorouting-state"; import { CursorRegistry, QueryHandlers, RevisionStore, type SessionSurface } from "../host/query"; import type { SdkFrame } from "../host/types"; import { @@ -4017,7 +4019,6 @@ export function createNotificationsExtension( controller?: NotificationSessionController; /** Whether this host mode can own the root SDK endpoint. Default: true. */ sdkHostModeSupported?: boolean; - onSdkRequest?: (kind: "control" | "query", connectionId: string, frame: Record) => void; runBtwTurn?: (question: string, signal: AbortSignal) => Promise<{ replyText: string }>; /** Observes settlement of optional session-branch startup after reconciliation completes. */ @@ -6826,6 +6827,7 @@ export function createNotificationsExtension( return { type: "query_response", ...response }; }, }); + if (isAutoroutingInactive(api)) markAutoroutingInactive(sdkRuntime.host); host = sdkRuntime.host; // Install the runtime before either transport can expose the host. session_start diff --git a/packages/coding-agent/src/sdk/bus/telegram-daemon-contract.ts b/packages/coding-agent/src/sdk/bus/telegram-daemon-contract.ts index 82daff3ede..9799d38768 100644 --- a/packages/coding-agent/src/sdk/bus/telegram-daemon-contract.ts +++ b/packages/coding-agent/src/sdk/bus/telegram-daemon-contract.ts @@ -274,7 +274,7 @@ export const SDK_LIFECYCLE_ROUTER_PROTOCOL_VERSION = 1; * Generation 169 delivers every ring-positioned session event live through the * bounded, capability-gated directed subscriber leg used by replay. */ -export const DAEMON_GENERATION = 171; +export const DAEMON_GENERATION = 172; /** * Serving-compatibility boundary for daemon lifecycle requests. Epoch 7 diff --git a/packages/coding-agent/src/sdk/host/host.ts b/packages/coding-agent/src/sdk/host/host.ts index f8b8819986..f8cc2736c9 100644 --- a/packages/coding-agent/src/sdk/host/host.ts +++ b/packages/coding-agent/src/sdk/host/host.ts @@ -1,6 +1,8 @@ import { logger } from "@gajae-code/utils"; +import { AUTOROUTING_INACTIVE_WARNING } from "../../config/autorouting-contract"; import { redactBrokerRuntimeCloseCapability } from "./control/runtime-gate"; import { type EventFrame, SessionEventStream } from "./events"; +import { isAutoroutingInactive } from "./internal-autorouting-state"; import { type ProviderLease, ReverseLeaseError, ReverseLeaseRuntime } from "./reverse-leases"; import type { BrokerIndexWriter, HostEndpointAdapters, SdkFrame } from "./types"; @@ -77,6 +79,7 @@ export interface SessionSdkHostOptions extends HostEndpointAdapters { connectionCapabilities?: (connectionId: string) => ReadonlySet | undefined; /** Readiness publication mode; defaults to the stock immediate contract. */ readiness?: SessionReadinessMode; + /** * Authorization for a deferred activation. It is consulted on every attempt * that would publish readiness and never on an idempotent replay, and a gate @@ -250,6 +253,17 @@ export class SessionSdkHost { async start(): Promise<"started" | "already"> { if (this.#started) return "already"; this.events.restart(); + if (isAutoroutingInactive(this)) { + this.emitEvent({ + kind: "notice", + payload: { + type: "notice", + level: "warning", + message: AUTOROUTING_INACTIVE_WARNING, + source: "autorouting", + }, + }); + } if (this.#options.readiness !== "deferred") this.#publishReadiness(); else this.emitEvent({ diff --git a/packages/coding-agent/src/sdk/host/internal-autorouting-state.ts b/packages/coding-agent/src/sdk/host/internal-autorouting-state.ts new file mode 100644 index 0000000000..96c33320a8 --- /dev/null +++ b/packages/coding-agent/src/sdk/host/internal-autorouting-state.ts @@ -0,0 +1,9 @@ +const inactiveAutoroutingState = new WeakMap(); + +export function markAutoroutingInactive(target: object): void { + inactiveAutoroutingState.set(target, true); +} + +export function isAutoroutingInactive(target: unknown): boolean { + return typeof target === "object" && target !== null && inactiveAutoroutingState.get(target) === true; +} diff --git a/packages/coding-agent/src/sdk/host/session-runtime.ts b/packages/coding-agent/src/sdk/host/session-runtime.ts index b3154fed02..ea49b9d5ae 100644 --- a/packages/coding-agent/src/sdk/host/session-runtime.ts +++ b/packages/coding-agent/src/sdk/host/session-runtime.ts @@ -46,6 +46,7 @@ import { import { type ControlSurface, controlRequestFromFrame, dispatchControl } from "./control"; import { BROKER_RUNTIME_CLOSE_CAPABILITY_FIELD } from "./control/runtime-gate"; import { SessionSdkHost, type SessionSdkHostOptions } from "./host"; +import { isAutoroutingInactive, markAutoroutingInactive } from "./internal-autorouting-state"; import { CursorRegistry, QueryHandlers, RevisionStore, type SessionSurface } from "./query"; import { createSdkCapabilities, @@ -386,9 +387,10 @@ export interface CreateSdkSessionRuntimeOptions { stateRoot: string; token: string; }): SessionSdkTransport | Promise; - onSdkRequest?: SessionSdkHostOptions["onRequest"]; /** Session settings; enables `config.patch` application on this runtime. */ settings?: Settings; + /** Callback for diagnostics and lifecycle request observation. */ + onSdkRequest?: SessionSdkHostOptions["onRequest"]; /** Mutable shadow of patched config values merged into query readback. */ configOverrides?: Map; /** Private session-owned terminal-abort capabilities; never exposed on ExtensionContext. */ @@ -3036,6 +3038,7 @@ export function createSdkSessionRuntimeExtension(api: ExtensionAPI, options: Cre if (request.operation === "session.close" && response.ok === true) ctx.shutdown(); }, }); + if (isAutoroutingInactive(api)) markAutoroutingInactive(runtime.host); const disposeGate = ctx.workflowGate?.onGateEmitted?.(gate => runtime.emitEvent({ kind: "workflow_gate", payload: gate }), ); diff --git a/packages/coding-agent/src/sdk/protocol/operation-inventory.generated.json b/packages/coding-agent/src/sdk/protocol/operation-inventory.generated.json index 43a2157e90..6b5487fc19 100644 --- a/packages/coding-agent/src/sdk/protocol/operation-inventory.generated.json +++ b/packages/coding-agent/src/sdk/protocol/operation-inventory.generated.json @@ -1854,6 +1854,17 @@ "packages/coding-agent/test/sdk-operation-inventory.test.ts" ] }, + { + "sourceId": "slash_command:routing", + "sourceFile": "packages/coding-agent/src/slash-commands/builtin-registry.ts", + "sourceKind": "slash_command", + "decision": "exclude", + "rationale": "visual/local-only autorouting settings toggle and smart-routing panel entry, not a user-facing SDK control seam", + "exclusionMetadata": { + "adapterMappings": "not_applicable", + "testIds": "not_applicable" + } + }, { "sourceId": "slash_command:export", "sourceFile": "packages/coding-agent/src/slash-commands/builtin-registry.ts", diff --git a/packages/coding-agent/src/sdk/session.ts b/packages/coding-agent/src/sdk/session.ts index 3403e9bc3e..dd7699eebd 100644 --- a/packages/coding-agent/src/sdk/session.ts +++ b/packages/coding-agent/src/sdk/session.ts @@ -52,6 +52,7 @@ import { import { loadCapability, reset as resetCapabilities } from "../capability"; import { type Rule, ruleCapability, setActiveRules } from "../capability/rule"; import type { SourceMeta } from "../capability/types"; +import { AUTOROUTING_INACTIVE_WARNING } from "../config/autorouting-contract"; import { resolveModelProfileName } from "../config/model-profile-contract"; import { resolveProfileBindings } from "../config/model-profiles"; import { kNoAuth, ModelRegistry } from "../config/model-registry"; @@ -136,6 +137,7 @@ import { import { createReconciliationStore, type ReconciliationStore } from "../sdk/bus/reconciliation-store"; import { NotificationSessionController } from "../sdk/bus/session-control"; import { shouldHostSdk } from "../sdk/host"; +import { markAutoroutingInactive } from "../sdk/host/internal-autorouting-state"; import { createSdkSessionRuntimeExtension, registerSdkOnlyNotificationCommand } from "../sdk/host/session-runtime"; import { createSdkWebSocketTransport } from "../sdk/host/websocket-transport"; import type { SecretObfuscator } from "../secrets"; @@ -1414,6 +1416,8 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} authStorage.setSessionCredentialSelector(scopeId, provider, selector); }; const settings = options.settings ?? (await logger.time("settings", Settings.init, { cwd, agentDir })); + const autoroutingInactive = + settings.get("task.autorouting.enabled") === true && !settings.getEffectiveAutorouting().active; // Cwd-derived runtime state must follow a rescope (`move_session`, `/move`), // so services resolve the LIVE session cwd per activation instead of // capturing the launch root. Before the manager exists the launch cwd is the @@ -3023,6 +3027,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} if (notificationsExtensionEligible || sdkHostEligible) { inlineExtensions.push(async api => { try { + if (autoroutingInactive) markAutoroutingInactive(api); if (lifecycleStartupCapability) attachLifecycleStartupCapability(api, lifecycleStartupCapability); if (lifecycleStartupCapability && process.env.GJC_SDK_TEST_FACTORY_FAILURE === cwd) throw new Error(process.env.GJC_SDK_TEST_FACTORY_SECRET ?? "Lifecycle factory test failure."); @@ -3030,6 +3035,7 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} const createNotificationsExtension = await notificationAdapterService.get("session-extension"); createNotificationsExtension(api, { settings, + controller: notificationSessionController, spawnedByGjc, sdkHostModeSupported: options.sdkHostModeSupported, @@ -4140,6 +4146,10 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} }); session.setActiveModelProfile(startupActiveModelProfile); session.configWarnings.push(...contextFileWarnings); + // Determined once, here, where settings are already available. Keep the + // durable warning for interactive and print consumers; ACP delivery is + // carried by the host replay ring through the internal runtime seam above. + if (autoroutingInactive) session.configWarnings.push(AUTOROUTING_INACTIVE_WARNING); hasSession = true; const sessionAsyncJobManager = asyncJobManager; if (sessionAsyncJobManager) { diff --git a/packages/coding-agent/src/session/artifacts.ts b/packages/coding-agent/src/session/artifacts.ts index 6304150b16..3e36e2c6e8 100644 --- a/packages/coding-agent/src/session/artifacts.ts +++ b/packages/coding-agent/src/session/artifacts.ts @@ -34,6 +34,13 @@ function isSafeFilename(filename: string): boolean { return /^[a-zA-Z0-9_.-]+$/.test(filename); } +const MAX_ATTEMPT_ID_LENGTH = 128; + +function assertSafeAttemptId(attemptId: string): void { + if (!/^[A-Za-z0-9_-]{1,128}$/.test(attemptId) || attemptId.length > MAX_ATTEMPT_ID_LENGTH) + throw new Error("Unsafe artifact attempt id"); +} + function parseManagedOutputGeneration(value: Uint8Array, outputFilenamePrefix: string): ManagedOutputGeneration | null { try { const parsed = JSON.parse(Buffer.from(value).toString("utf8")) as Partial; @@ -99,13 +106,29 @@ export class ArtifactManager { #dirCreated = false; #initialized: Promise | undefined; #initializedComplete = false; + readonly #attemptId: string | undefined; + readonly #allocatedIds = new Set(); + readonly #retiredIds = new Set(); + #reservations = new Map(); + readonly #stagingParentStore: ManagedSessionDescendantStore | undefined; + readonly #stagingRelativePath: string | undefined; /** * @param dir Directory that will hold artifact files. Created lazily on first save. */ - constructor(target: string | ManagedSessionDescendantStore) { + constructor( + target: string | ManagedSessionDescendantStore, + options?: { + readonly attemptId?: string; + readonly stagingParentStore?: ManagedSessionDescendantStore; + readonly stagingRelativePath?: string; + }, + ) { this.#store = typeof target === "string" ? undefined : target; this.#dir = typeof target === "string" ? target : target.dir; + this.#attemptId = options?.attemptId; + this.#stagingParentStore = options?.stagingParentStore; + this.#stagingRelativePath = options?.stagingRelativePath; } /** @@ -188,8 +211,12 @@ export class ArtifactManager { } async #publish(content: string, filename: string): Promise { - if (this.#store) await this.#store.publishNoReplace(filename, Buffer.from(content, "utf8")); - else await publishManagedFileNoReplace(path.join(this.#dir, filename), Buffer.from(content, "utf8")); + await this.#publishBytes(Buffer.from(content, "utf8"), filename); + } + + async #publishBytes(bytes: Uint8Array, filename: string): Promise { + if (this.#store) await this.#store.publishNoReplace(filename, bytes); + else await publishManagedFileNoReplace(path.join(this.#dir, filename), bytes); } async replaceNamed(filename: string, content: string): Promise { @@ -289,8 +316,19 @@ export class ArtifactManager { if (!/^[a-zA-Z0-9_.-]+$/.test(filename)) return false; try { if (this.#store) { + const beforePaths = new Set(await fs.readdir(this.#store.dir)); const staged = this.#store.readExpected(filename); if (staged) this.#store.removeExpected(filename, staged); + // Capture only root names outside retained authority so foreign sibling + // placeholders cannot make rollback fail before exact removal runs. + for (const basename of await fs.readdir(this.#store.dir)) { + const nativeResidue = /^\.gjc-/u.test(basename); + const ownQuarantine = basename === `${filename}.removing` || basename.startsWith(`${filename}.`); + if (beforePaths.has(basename) || (!nativeResidue && !ownQuarantine)) continue; + const residuePath = path.join(this.#store.dir, basename); + const stat = await fs.lstat(residuePath); + await fs.rm(residuePath, { recursive: stat.isDirectory(), force: true }); + } } else { await fs.unlink(path.join(this.#dir, filename)); } @@ -326,7 +364,10 @@ export class ArtifactManager { * Prefer `allocatePath` or `save`; this synchronous seam exists for pruning callbacks. */ allocateId(): number { - return this.#claimNextIdSync(); + while (this.#retiredIds.has(String(this.#nextId))) this.#nextId++; + const id = this.#claimNextIdSync(); + this.#allocatedIds.add(String(id)); + return id; } /** @@ -338,6 +379,7 @@ export class ArtifactManager { async allocatePath(toolType: string): Promise<{ id: string; path?: string }> { await this.#ensureDir(); const id = String(await this.#claimNextId()); + this.#allocatedIds.add(id); if (this.#store) return { id }; return { id, path: path.join(this.#dir, this.#filename(id, toolType)) }; } @@ -350,6 +392,7 @@ export class ArtifactManager { async save(content: string, toolType: string, options: ArtifactSaveOptions = {}): Promise { await this.#ensureDir(); const id = String(await this.#claimNextId()); + this.#allocatedIds.add(id); const maxBytes = Math.max(0, options.maxBytes ?? DEFAULT_ARTIFACT_MAX_BYTES); const contentBytes = Buffer.byteLength(content, "utf-8"); const published = @@ -378,11 +421,198 @@ export class ArtifactManager { */ async listFiles(): Promise { try { + if (this.#store) { + return this.#store + .captureTree("") + .entries.filter(entry => entry.kind === "file" && entry.relativePath.length > 0) + .map(entry => entry.relativePath); + } return await fs.readdir(this.#dir); } catch { return []; } } + getAttemptId(): string | undefined { + return this.#attemptId; + } + + getAllocatedIds(): readonly string[] { + return [...this.#allocatedIds].sort((a, b) => Number(a) - Number(b)); + } + + /** Create an isolated artifact manager rooted below this manager's staging area. */ + createAttemptStaging(attemptId: string): ArtifactManager { + assertSafeAttemptId(attemptId); + const stagingRelativePath = path.posix.join(".staging", attemptId); + const target = this.#store + ? this.#store.deriveSubtree(stagingRelativePath) + : path.join(this.#dir, stagingRelativePath); + return new ArtifactManager(target, { + attemptId, + ...(this.#store ? { stagingParentStore: this.#store, stagingRelativePath } : {}), + }); + } + + /** Reserve and publish a candidate's staged artifacts with a contiguous parent ID block. */ + async commitAttemptStaging( + staging: ArtifactManager, + attemptId: string, + options?: { beforePublish?: (mapping: ReadonlyMap) => Promise | void }, + ): Promise> { + if (staging.#attemptId !== attemptId) throw new Error("Artifact staging ownership mismatch"); + assertSafeAttemptId(attemptId); + await this.#ensureDir(); + await staging.#ensureDir(); + const ids = staging.getAllocatedIds(); + const start = this.#nextId; + this.#nextId += ids.length; + const mapping = new Map(); + for (let index = 0; index < ids.length; index++) mapping.set(ids[index]!, String(start + index)); + const frozenMap = new Map(mapping) as Map & ReadonlyMap; + Object.defineProperties(frozenMap, { + set: { + value: () => { + throw new Error("Artifact ID map is immutable"); + }, + }, + delete: { + value: () => { + throw new Error("Artifact ID map is immutable"); + }, + }, + clear: { + value: () => { + throw new Error("Artifact ID map is immutable"); + }, + }, + }); + Object.freeze(frozenMap); + const publishedNames: string[] = []; + try { + await options?.beforePublish?.(frozenMap); + for (const filename of await staging.listFiles()) { + if (/^\.artifact-id-\d+$/.test(filename)) continue; + const bytes = staging.#store + ? staging.#store.readExpected(filename)?.bytes + : await fs.readFile(path.join(staging.#dir, filename)); + if (!bytes) continue; + const match = filename.match(/^(\d+)(\..*)$/); + const mappedFilename = match ? `${mapping.get(match[1]!) ?? match[1]}${match[2]}` : filename; + await this.#publishBytes(bytes, mappedFilename); + publishedNames.push(mappedFilename); + } + this.#reservations.set(attemptId, { start, count: ids.length, names: [...publishedNames] }); + await staging.discardAttemptStaging(); + return frozenMap; + } catch (error) { + // Cleanup is best-effort, but a durable removal failure must never be silent: it leaves a + // published artifact behind under an id we are about to retire. Surface it alongside the + // original publication error rather than dropping the boolean. + const unremoved: string[] = []; + for (const filename of publishedNames.reverse()) + if (!(await this.removeNamedBestEffort(filename))) unremoved.push(filename); + // Only rewind the tail when every published artifact was actually removed. If any removal + // failed, the file still occupies its id, so retire the whole block instead of handing the + // ids out again. + if (unremoved.length === 0 && this.#nextId === start + ids.length) this.#nextId = start; + else for (const id of mapping.values()) this.#retiredIds.add(id); + if (unremoved.length > 0) + throw new AggregateError( + [error, new Error(`Failed to roll back published artifacts: ${unremoved.join(", ")}`)], + "Attempt-staging publication failed and rollback left artifacts behind.", + ); + throw error; + } + } + async rollbackLastAttemptCommit(attemptId?: string): Promise { + if (attemptId === undefined) return; + const reservation = this.#reservations.get(attemptId); + if (!reservation) return; + const unremoved: string[] = []; + for (const filename of [...reservation.names].reverse()) + if (!(await this.removeNamedBestEffort(filename))) unremoved.push(filename); + // Only rewind the tail when every published artifact was actually removed; otherwise retire the + // whole block so a leaked file's id can never be reallocated. + if (unremoved.length === 0 && this.#nextId === reservation.start + reservation.count) + this.#nextId = reservation.start; + else + for (let index = 0; index < reservation.count; index++) + this.#retiredIds.add(String(reservation.start + index)); + this.#reservations.delete(attemptId); + // The reservation is always released so ids can never be reused, but a failed durable removal + // is reported rather than swallowed. + if (unremoved.length > 0) throw new Error(`Failed to roll back published artifacts: ${unremoved.join(", ")}`); + } + + finalizeLastAttemptCommit(attemptId?: string): void { + if (attemptId !== undefined) this.#reservations.delete(attemptId); + } + + async discardAttemptStaging(): Promise { + if (this.#store) { + const cleanupStore = this.#stagingParentStore ?? this.#store; + const cleanupPath = this.#stagingParentStore ? this.#stagingRelativePath! : ""; + const parentCleanupPath = cleanupPath ? path.posix.dirname(cleanupPath) : ""; + let parentBefore: ReturnType | undefined; + if (this.#stagingParentStore) { + try { + parentBefore = cleanupStore.captureTree(parentCleanupPath); + } catch { + parentBefore = undefined; + } + } + try { + const snapshot = cleanupStore.captureTree(cleanupPath); + cleanupStore.removeTreeExpected(cleanupPath, snapshot); + } catch (error) { + if (!(error instanceof Error && (error.message === "not_found" || error.message === "cleanup_pending"))) + throw error; + } + if (this.#stagingParentStore && parentBefore) { + try { + const after = cleanupStore.captureTree(parentCleanupPath); + const beforePaths = new Set(parentBefore.entries.map(entry => entry.relativePath)); + for (const entry of after.entries) { + if ( + entry.kind === "directory" && + entry.relativePath.length > 0 && + !beforePaths.has(entry.relativePath) && + /\.removing$/u.test(path.posix.basename(entry.relativePath)) + ) { + await fs.rm(path.join(cleanupStore.dir, parentCleanupPath, entry.relativePath), { + recursive: true, + force: true, + }); + continue; + } + if ( + entry.kind !== "file" || + beforePaths.has(entry.relativePath) || + !/^\\.gjc-(?:exact-unlink-placeholder|remove)-/u.test(path.posix.basename(entry.relativePath)) + ) + continue; + const relative = path.posix.join(parentCleanupPath, entry.relativePath); + const expected = cleanupStore.readExpected(relative); + if (expected) { + try { + cleanupStore.removeExpected(relative, expected); + } catch (cleanupError) { + if (!(cleanupError instanceof Error && cleanupError.message === "cleanup_pending")) + throw cleanupError; + } + } + await fs.rm(path.join(cleanupStore.dir, relative), { force: true }).catch(() => undefined); + } + } catch { + // Retained cleanup evidence is safe to leave for a later maintenance pass. + } + } + this.#store.close(); + } else { + await fs.rm(this.#dir, { recursive: true, force: true }); + } + this.#dirCreated = false; + } /** Persist exact UTF-8 text for heap-eviction rehydration. */ async publishExactText(text: string, options: ArtifactPublishOptions = {}): Promise { diff --git a/packages/coding-agent/src/session/internal/managed-session-storage.ts b/packages/coding-agent/src/session/internal/managed-session-storage.ts index dbcaa1e7e9..a427564b34 100644 --- a/packages/coding-agent/src/session/internal/managed-session-storage.ts +++ b/packages/coding-agent/src/session/internal/managed-session-storage.ts @@ -176,6 +176,7 @@ function sameDirectoryTreeSnapshotAfterRename( ); } +/** A managed move outcome that reports whether the mutation is known not to have committed. */ export class ManagedTreeMoveOutcomeError extends Error { constructor( message: string, @@ -2390,6 +2391,73 @@ export class ManagedSessionDescendantStore { return movedSnapshot.snapshot; } + /** Move one exact managed regular file to an absent destination through retained authority. */ + moveFileNoReplace( + sourceRelativePath: string, + destinationRelativePath: string, + expected: ManagedFileSnapshot, + options?: { + /** Subtree store bound to the source's directory; keeps the verification read basename-scoped when this store lacks retained authority. */ + sourceStore: ManagedSessionDescendantStore; + sourceStoreRelativePath: string; + }, + ): ManagedFileSnapshot { + this.#assertBound(); + const sourceResolved = this.#resolve(sourceRelativePath); + const destinationResolved = this.#resolve(destinationRelativePath); + if ( + options && + path.resolve(options.sourceStore.#resolve(options.sourceStoreRelativePath)) !== path.resolve(sourceResolved) + ) + throw new ManagedTreeMoveOutcomeError("artifact_source_changed", false); + const source = options + ? options.sourceStore.readExpected(options.sourceStoreRelativePath) + : this.readExpected(sourceRelativePath); + if ( + !source || + !sameReplacementIdentity(source.identity, expected.identity) || + source.identity.sha256 !== expected.identity.sha256 || + !source.bytes.equals(expected.bytes) + ) + throw new ManagedTreeMoveOutcomeError("artifact_source_changed", false); + + const moved = this.#authority + ? this.#authority.renameManagedFileNoReplace( + this.#relative(sourceResolved), + this.#relative(destinationResolved), + expected.identity.dev.toString(), + expected.identity.ino.toString(), + expected.identity.size.toString(), + expected.identity.mtimeNs.toString(), + expected.identity.ctimeNs.toString(), + expected.identity.sha256, + ) + : nativeSessionStorage().renameNoReplacePath(sourceResolved, destinationResolved); + const outcome = classifyNativePublishOutcome(moved, this.#authority ? "retained_file" : "direct_rename"); + if (!outcome.ok) + throw new ManagedTreeMoveOutcomeError(publishFailure(outcome).message, mayCleanCurrentStaging(outcome)); + + const movedSnapshot = this.readExpected(destinationRelativePath); + if ( + !movedSnapshot || + !sameReplacementIdentity(movedSnapshot.identity, expected.identity) || + movedSnapshot.identity.sha256 !== expected.identity.sha256 || + !movedSnapshot.bytes.equals(expected.bytes) + ) + throw new ManagedTreeMoveOutcomeError("artifact_destination_mismatch", false); + if (!this.#authority) { + try { + fsyncDirectory(path.dirname(sourceResolved)); + if (path.dirname(sourceResolved) !== path.dirname(destinationResolved)) + fsyncDirectory(path.dirname(destinationResolved)); + } catch { + throw new ManagedTreeMoveOutcomeError("managed_publish_fsync_failed", false); + } + } + this.#assertBound(); + return movedSnapshot; + } + removeTreeExpected(relativePath: string, expected: NativeDirectoryTreeSnapshot): void { this.#beforeMutation(); this.#assertBound(); diff --git a/packages/coding-agent/src/session/session-manager.ts b/packages/coding-agent/src/session/session-manager.ts index fa5bd6625d..119ed2ba3d 100644 --- a/packages/coding-agent/src/session/session-manager.ts +++ b/packages/coding-agent/src/session/session-manager.ts @@ -202,6 +202,7 @@ import { stripInternalDetailsFields, } from "./messages"; import { type SessionManagerReadAccess, sessionManagerReadCapability } from "./session-manager-internal"; +import { isStagedSessionPath, SESSION_STAGING_DIRNAME } from "./session-staging-paths"; import type { ManagedSessionSecurityContext, SessionStorage, @@ -289,6 +290,103 @@ interface PreparedNewSessionState extends PreparedNewSession { type ResidentTransitionFailurePolicy = "install-staged" | "memory-fallback" | "retain-and-throw" | "memory-only"; type ResidentBlobMissingPolicy = "throw" | "placeholder"; +const MAX_STAGED_ATTEMPT_ID_LENGTH = 128; + +function assertSafeStagedAttemptId(attemptId: string): void { + if (!/^[A-Za-z0-9_-]{1,128}$/.test(attemptId) || attemptId.length > MAX_STAGED_ATTEMPT_ID_LENGTH) + throw new Error("Unsafe artifact attempt id"); +} + +const ATTEMPT_REMAP_STRUCTURAL_KEYS = new Set(["id", "parentId", "timestamp"]); +const ARTIFACT_REFERENCE_KEYS = new Set([ + "artifactId", + "artifactIds", + "artifactRef", + "artifactRefs", + "agentId", + "agentIds", + "agentRef", + "agentRefs", +]); +/** + * Trailing-selector grammar mirrored from `splitPathAndSel` / `splitInternalUrlSel` in + * `src/tools/path-utils.ts`. It is duplicated rather than imported because `path-utils` pulls in + * `internal-urls`, which reaches back into the session layer and would create an import cycle. + * The boundary red-team suite cross-checks these against the real parser so drift fails a test. + */ +const SELECTOR_RANGE_RE = /^L?\d+(?:[-+]L?\d+|-)?(?:,L?\d+(?:[-+]L?\d+|-)?)*$/i; +const SELECTOR_TAIL_RE = /^(?:L?\d+(?:[-+]L?\d+|-)?(?:,L?\d+(?:[-+]L?\d+|-)?)*|raw|conflicts)$/i; + +/** + * Decide whether a `:` directly after `://` terminates the id (so the id is a remappable + * reference and the tail is an opaque selector) or is part of an opaque authority (so the id must be + * left alone). `artifact://` splits unconditionally at the first colon; every other scheme requires a + * strict selector tail, so `agent://3:bogus` keeps `3:bogus` as the authority and must NOT be remapped. + */ +function colonTerminatesUriId(scheme: string, tail: string): boolean { + if (scheme.toLowerCase() === "artifact") return true; + if (SELECTOR_TAIL_RE.test(tail)) return true; + const innerColon = tail.lastIndexOf(":"); + if (innerColon <= 0) return false; + const head = tail.slice(0, innerColon); + const last = tail.slice(innerColon + 1); + const headIsRaw = /^raw$/i.test(head); + const lastIsRaw = /^raw$/i.test(last); + return (headIsRaw && SELECTOR_RANGE_RE.test(last)) || (SELECTOR_RANGE_RE.test(head) && lastIsRaw); +} + +function remapArtifactReferenceString(value: string, idMap: ReadonlyMap, exactId = false): string { + const exact = exactId ? idMap.get(value) : undefined; + if (exact !== undefined) return exact; + // Only re-key when the ENTIRE value is a single URI reference token. A string carrying prose or + // multiple tokens is opaque content we do not own, and rewriting inside it caused real corruption + // in earlier revisions. Everything after the id (selector, query, fragment) is likewise opaque and + // is preserved verbatim, so nested ids inside those payloads are never re-keyed. + if (/[\s"'<>]/.test(value)) return value; + const head = value.match(/^(artifact|agent):\/\/([0-9]+)/i); + if (!head) return value; + const protocol = head[1]; + const mapped = idMap.get(head[2]); + if (mapped === undefined) return value; + const rest = value.slice(head[0].length); + if (rest !== "" && !/^[:/?#]/.test(rest)) return value; + if (rest.startsWith(":") && !colonTerminatesUriId(protocol, rest.slice(1))) return value; + return `${protocol}://${mapped}${rest}`; +} + +function remapAttemptReferencesInEntries(entries: readonly FileEntry[], idMap: ReadonlyMap): void { + if (idMap.size === 0) return; + const seen = new WeakSet(); + const visit = (value: unknown, key?: string): unknown => { + if ( + typeof value === "number" && + key !== undefined && + ARTIFACT_REFERENCE_KEYS.has(key) && + Number.isSafeInteger(value) + ) { + const mapped = idMap.get(String(value)); + return mapped === undefined ? value : Number(mapped); + } + if (typeof value === "string") { + return remapArtifactReferenceString(value, idMap, key !== undefined && ARTIFACT_REFERENCE_KEYS.has(key)); + } + if (!value || typeof value !== "object" || seen.has(value)) return value; + seen.add(value); + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index++) value[index] = visit(value[index], key); + return value; + } + const record = value as Record; + for (const [childKey, child] of Object.entries(record)) { + if (ATTEMPT_REMAP_STRUCTURAL_KEYS.has(childKey)) continue; + const next = visit(child, childKey); + if (next !== child) record[childKey] = next; + } + return value; + }; + for (const entry of entries) visit(entry); +} + type ResidentTransitionSource = | { mode: "materialize"; @@ -3403,6 +3501,7 @@ async function readTerminalBreadcrumb(cwd: string): Promise { return null; } + if (isStagedSessionPath(sessionFile)) return null; const inspected = inspectResumeSessionFile(sessionFile, new FileSessionStorage()); if ("kind" in inspected) { if (inspected.reason !== "missing") return null; @@ -3416,7 +3515,8 @@ async function readTerminalBreadcrumb(cwd: string): Promise { const listing = listManagedCandidates(resolved.scope); if (listing.kind !== "complete") return null; const migrated = listing.owned.filter( - candidate => path.basename(candidate.path) === path.basename(sessionFile), + candidate => + !isStagedSessionPath(candidate.path) && path.basename(candidate.path) === path.basename(sessionFile), ); return migrated.length === 1 ? migrated[0]!.path : null; } @@ -3431,9 +3531,14 @@ async function readTerminalBreadcrumb(cwd: string): Promise { if (resolved.kind !== "resolved") return null; const listing = listManagedCandidates(resolved.scope); if (listing.kind !== "complete") return null; - const exact = listing.owned.find(candidate => path.resolve(candidate.path) === path.resolve(sessionFile)); + const exact = listing.owned.find( + candidate => + !isStagedSessionPath(candidate.path) && path.resolve(candidate.path) === path.resolve(sessionFile), + ); if (exact) return exact.path; - const byIdentity = listing.owned.find(candidate => candidate.sessionId === header.id); + const byIdentity = listing.owned.find( + candidate => !isStagedSessionPath(candidate.path) && candidate.sessionId === header.id, + ); if (byIdentity) return byIdentity.path; return pathIsWithin(sessionsRoot, path.resolve(sessionFile)) ? null : path.resolve(sessionFile); } catch (err) { @@ -6245,9 +6350,11 @@ const PROJECT_SESSION_SCAN_MAX_DIRECTORIES = 4096; const PROJECT_SESSION_SCAN_MAX_FILES = 1000; function isProjectSessionTranscriptPath(projectGjcDir: string, filePath: string): boolean { + if (isStagedSessionPath(filePath)) return false; const relative = path.relative(projectGjcDir, filePath); if (relative.startsWith("..") || path.isAbsolute(relative)) return false; const segments = relative.split(path.sep); + if (segments.includes(SESSION_STAGING_DIRNAME)) return false; if (segments.length === 1) return true; const parent = segments.at(-2); return parent === "agent-session" || segments.includes("sessions"); @@ -6258,7 +6365,7 @@ function isProjectSessionTranscriptPath(projectGjcDir: string, filePath: string) * Runtime token/audit JSONL files are excluded by requiring a known transcript * container (`agent-session` or `sessions`). */ -function listProjectSessionTranscriptFiles(cwd: string): string[] { +export function listProjectSessionTranscriptFiles(cwd: string): string[] { const projectGjcDir = path.join(path.resolve(cwd), ".gjc"); let rootStat: fs.Stats; try { @@ -6284,6 +6391,7 @@ function listProjectSessionTranscriptFiles(cwd: string): string[] { if (entry.isSymbolicLink()) continue; const entryPath = path.join(directory, entry.name); if (entry.isDirectory()) { + if (entry.name === SESSION_STAGING_DIRNAME) continue; directories.push(entryPath); continue; } @@ -7205,6 +7313,31 @@ export class SessionManager { /** Publication fence counter carried by the mutable `.spill.commit` marker. */ #commitGen = 0; /** Failed staged persistence retains its exact writer and temporary pathname for retryable cleanup. */ + /** Candidate-owned session publication state; set only by staged factories. */ + #stagedPublication: + | { + finalSessionFile: string; + stagedSessionFile: string; + finalDestination: SessionDestination; + managedParentStore?: ManagedSessionDescendantStore; + /** Subtree store bound to the staging directory; basename reads stay authority-safe. */ + managedStagingStore?: ManagedSessionDescendantStore; + attemptId: string; + committed: boolean; + discarded: boolean; + publishedFinalSnapshot?: ManagedFileSnapshot; + publishedFinalBytes?: Buffer; + deferArtifactFinalize?: boolean; + /** + * A publish that native code could neither prove nor disprove. The transcript may + * already be visible at the destination, so no later cleanup may reclaim the + * staging or artifacts it references. + */ + preservedUnproven?: boolean; + } + | undefined; + #stagedArtifactParent: ArtifactManager | null = null; + #stagedCommitArtifactParent: ArtifactManager | null = null; #preparedNewSessionCleanupInProgress = false; /** Active cold-sidecar runtime (retirement + lazy resolution). Undefined when disabled. */ #sidecarRuntime: SessionMemorySidecarRuntime | undefined = undefined; @@ -9161,7 +9294,7 @@ export class SessionManager { this.#effectiveSessionMemoryMode(transcriptSize) === "enabled" && this.#storage.existsSync(`${sidecarRoot}/.session-memory.spill.commit`); if (boundedTranscriptAdmitted && (await this.#tryInitSessionFileFromSidecar(resolvedSessionFile))) { - writeTerminalBreadcrumb(this.cwd, resolvedSessionFile); + this.#writeTerminalBreadcrumb(resolvedSessionFile); revalidateStrictResume(); return; } @@ -9171,7 +9304,7 @@ export class SessionManager { this.#sessionMemoryMode = "shadow"; this.#sessionMemoryAutoDisabledReason = "sidecar_reload_failures"; } - writeTerminalBreadcrumb(this.cwd, resolvedSessionFile); + this.#writeTerminalBreadcrumb(resolvedSessionFile); revalidateStrictResume(); return; } @@ -9187,7 +9320,7 @@ export class SessionManager { this.#applyFreshSessionMetadata(fresh); this.#commitResidentTextStoreTransition(prepared); this.#retireEphemeralArtifacts(); - writeTerminalBreadcrumb(this.cwd, resolvedSessionFile); + this.#writeTerminalBreadcrumb(resolvedSessionFile); await this.#rewriteFile(); this.#flushed = true; this.#ensuredOnDisk = true; @@ -9215,7 +9348,7 @@ export class SessionManager { this.#applyFreshSessionMetadata(fresh); this.#commitResidentTextStoreTransition(prepared); this.#retireEphemeralArtifacts(); - writeTerminalBreadcrumb(this.cwd, resolvedSessionFile); + this.#writeTerminalBreadcrumb(resolvedSessionFile); await this.#rewriteFile(); this.#flushed = true; this.#ensuredOnDisk = true; @@ -9248,7 +9381,7 @@ export class SessionManager { this.#titleSource = header?.titleSource; this.#needsFullRewriteOnNextPersist = migrationApplied; this.#commitResidentTextStoreTransition(prepared); - writeTerminalBreadcrumb(this.cwd, resolvedSessionFile); + this.#writeTerminalBreadcrumb(resolvedSessionFile); this.#flushed = true; this.#ensuredOnDisk = true; this.#adoptManagedPersistIdentity(resolvedSessionFile); @@ -9260,6 +9393,11 @@ export class SessionManager { } } + #writeTerminalBreadcrumb(sessionFile: string): void { + if (this.#stagedPublication && !this.#stagedPublication.committed) return; + writeTerminalBreadcrumb(this.cwd, sessionFile); + } + async #hydrateExistingSession( sessionFile: string, entries: FileEntry[], @@ -16036,8 +16174,9 @@ export class SessionManager { * Adopt an externally-owned ArtifactManager. Used by subagents to share * the parent session's artifact directory and ID counter. */ - adoptArtifactManager(manager: ArtifactManager): void { + adoptArtifactManager(manager: ArtifactManager, parent?: ArtifactManager): void { this.#adoptedArtifactManager = manager; + if (parent) this.#stagedArtifactParent = parent; } /** Release only the matching externally adopted manager. */ @@ -17241,6 +17380,15 @@ export class SessionManager { } } + /** Remap artifact references in an unpublished candidate before its publication fence. */ + async remapStagedArtifactReferences(idMap: ReadonlyMap): Promise { + const staged = this.#stagedPublication; + if (!staged || staged.committed || staged.discarded) throw new Error("Staged session is unavailable"); + remapAttemptReferencesInEntries(this.#fileEntries, idMap); + this.#needsFullRewriteOnNextPersist = true; + await this.#rewriteFileContents(); + } + /** * Append a custom message entry (for extensions) that participates in LLM context. * @param customType Hook identifier for filtering on reload @@ -18100,7 +18248,7 @@ export class SessionManager { async #sanitizeLoadedOpenAIResponsesReplayMetadataAndPersist(): Promise { const patches = this.#sanitizeLoadedOpenAIResponsesReplayMetadata(); - await this.#persistPatches(patches); + if (!this.isManagedDestination()) await this.#persistPatches(patches); return patches.length > 0; } @@ -19173,6 +19321,383 @@ export class SessionManager { * @param path Path to session file * @param sessionDir Optional session directory for /new or /branch. If omitted, derives from file's parent. */ + /** Open an unpublished candidate transcript below the reserved staging directory. */ + static async openStaged( + finalSessionFile: string, + storage: SessionStorage = new FileSessionStorage(), + attemptId: string = crypto.randomUUID(), + ): Promise { + assertSafeStagedAttemptId(attemptId); + + if (isStagedSessionPath(finalSessionFile)) throw new Error("Final session path cannot be staged"); + const finalPath = + storage instanceof FileSessionStorage ? canonicalizeTrustedPath(finalSessionFile) : finalSessionFile; + const finalDestination = explicitDestination(path.dirname(finalPath)); + const stagingDir = path.join(path.dirname(finalPath), SESSION_STAGING_DIRNAME); + await fs.promises.mkdir(stagingDir, { recursive: true, mode: 0o700 }); + const stagedSessionFile = path.join(stagingDir, `${attemptId}.jsonl`); + if (path.resolve(stagedSessionFile) === path.resolve(finalPath)) + throw new Error("Staged session path collides with final transcript"); + + const manager = new SessionManager(getProjectDir(), stagingDir, true, storage, finalDestination); + manager.#stagedPublication = { + finalSessionFile: finalPath, + stagedSessionFile, + finalDestination, + attemptId, + committed: false, + discarded: false, + }; + if (fs.existsSync(stagedSessionFile)) throw new Error("Staged session attempt already exists"); + try { + await manager.#initSessionFile(stagedSessionFile, true); + const parentArtifacts = new ArtifactManager(finalPath.endsWith(".jsonl") ? finalPath.slice(0, -6) : finalPath); + manager.adoptArtifactManager(parentArtifacts.createAttemptStaging(attemptId), parentArtifacts); + return manager; + } catch (error) { + try { + await manager.discardStaged(); + } catch (cleanupError) { + throw new AggregateError( + [toError(error), toError(cleanupError)], + "Staged session open and cleanup both failed.", + ); + } + throw error; + } + } + + /** Managed-authority variant of {@link openStaged}. */ + static async stagedNestedManaged( + finalSessionFile: string, + destination: SessionDestination, + store: ManagedSessionDescendantStore, + storage: SessionStorage = new FileSessionStorage(), + attemptId: string = crypto.randomUUID(), + ): Promise { + assertSafeStagedAttemptId(attemptId); + + if (destination.kind !== "managed" || !trustedSessionDestinations.has(destination)) + throw new Error("Nested managed session authority is unavailable"); + if (isStagedSessionPath(finalSessionFile)) throw new Error("Final session path cannot be staged"); + store.assertBound(); + const finalPath = path.resolve(finalSessionFile); + if (path.dirname(finalPath) !== path.resolve(destination.directory)) + throw new Error("Nested managed session escaped retained authority"); + const stagingStore = store.deriveSubtree(SESSION_STAGING_DIRNAME); + const stagedDir = stagingStore.dir; + const stagedSessionFile = path.join(stagedDir, `${attemptId}.jsonl`); + if (path.resolve(stagedSessionFile) === path.resolve(finalPath)) + throw new Error("Staged session path collides with final transcript"); + const stagedDestination = SessionManager.nestedManagedDestination(stagingStore, stagedDir); + const manager = new SessionManager(getProjectDir(), stagedDir, true, storage, stagedDestination); + manager.#stagedPublication = { + finalSessionFile: finalPath, + stagedSessionFile, + finalDestination: destination, + managedParentStore: store, + managedStagingStore: stagingStore, + attemptId, + committed: false, + discarded: false, + }; + if (fs.existsSync(stagedSessionFile)) throw new Error("Staged session attempt already exists"); + try { + await manager.#initSessionFile(stagedSessionFile, true); + const parentArtifacts = new ArtifactManager(store); + manager.adoptArtifactManager(parentArtifacts.createAttemptStaging(attemptId), parentArtifacts); + store.assertBound(); + return manager; + } catch (error) { + try { + await manager.discardStaged(); + } catch (cleanupError) { + throw new AggregateError( + [toError(error), toError(cleanupError)], + "Staged session open and cleanup both failed.", + ); + } + throw error; + } + } + /** Publish the candidate-owned staged transcript and artifacts at the real accept fence. */ + static async openStagedNestedManaged( + finalSessionFile: string, + destination: SessionDestination, + store: ManagedSessionDescendantStore, + storage: SessionStorage = new FileSessionStorage(), + attemptId: string = crypto.randomUUID(), + ): Promise { + return SessionManager.stagedNestedManaged(finalSessionFile, destination, store, storage, attemptId); + } + + async commitStaged(options?: { deferArtifactFinalize?: boolean }): Promise { + const staged = this.#stagedPublication; + if (!staged || staged.discarded) throw new Error("Staged session is unavailable"); + if (staged.committed) return; + staged.deferArtifactFinalize = options?.deferArtifactFinalize === true; + await this.flush(); + await this.#closePersistWriter(); + const stagedManager = this.#adoptedArtifactManager ?? this.#artifactManager; + // Lifecycle invariant: while a staged publication is uncommitted, the session must + // carry exactly the attempt-rooted staging manager it opened with. openStaged, + // stagedNestedManaged, and openStagedSession all pre-adopt an attempt-rooted + // manager, so an adopted manager that is absent or foreign means a second root + // replaced the first — orphaning the original from commit/discard cleanup. + if (this.#adoptedArtifactManager?.getAttemptId() !== staged.attemptId) + throw new Error("Staged session artifact root does not match the staged attempt."); + const parentArtifacts = + this.#stagedArtifactParent ?? + new ArtifactManager( + staged.finalSessionFile.endsWith(".jsonl") ? staged.finalSessionFile.slice(0, -6) : staged.finalSessionFile, + ); + this.#stagedCommitArtifactParent = parentArtifacts; + let published = false; + try { + if (stagedManager?.getAttemptId() === staged.attemptId) { + const stagedArtifactFiles = await stagedManager.listFiles(); + if (stagedArtifactFiles.length > 0 || stagedManager.getAllocatedIds().length > 0) { + await parentArtifacts.commitAttemptStaging(stagedManager, staged.attemptId, { + beforePublish: idMap => this.remapStagedArtifactReferences(idMap), + }); + } else await stagedManager.discardAttemptStaging(); + } + if (staged.managedParentStore && staged.managedStagingStore) { + const stagedName = path.basename(staged.stagedSessionFile); + const relative = path.posix.join(SESSION_STAGING_DIRNAME, stagedName); + const expected = staged.managedStagingStore.readExpected(stagedName); + if (!expected) throw new Error("staged_session_missing"); + staged.managedParentStore.moveFileNoReplace(relative, path.basename(staged.finalSessionFile), expected, { + sourceStore: staged.managedStagingStore, + sourceStoreRelativePath: stagedName, + }); + } else { + const outcome = classifyNativePublishOutcome( + nativeSessionManager().renameNoReplacePath(staged.stagedSessionFile, staged.finalSessionFile), + ); + if (!outcome.ok) throw new Error(outcome.code ?? "staged_session_publish_failed"); + } + published = true; + if (staged.managedParentStore) { + staged.publishedFinalSnapshot = + staged.managedParentStore.readExpected(path.basename(staged.finalSessionFile)) ?? undefined; + if (!staged.publishedFinalSnapshot) throw new Error("staged_session_publish_missing"); + } else staged.publishedFinalBytes = await fs.promises.readFile(staged.finalSessionFile); + + const finalStat = fs.lstatSync(staged.finalSessionFile, { bigint: true }); + if (!finalStat.isFile() || finalStat.isSymbolicLink()) throw new Error("staged_session_identity_changed"); + this.sessionDir = path.dirname(staged.finalSessionFile); + this.destination = staged.finalDestination; + this.#managedTranscriptStoreCache = staged.managedParentStore + ? { directory: path.dirname(staged.finalSessionFile), store: staged.managedParentStore } + : null; + this.#sessionFile = staged.finalSessionFile; + this.#artifactManager = parentArtifacts; + this.#artifactManagerSessionFile = staged.finalSessionFile; + this.#adoptedArtifactManager = parentArtifacts; + staged.committed = true; + if (!staged.deferArtifactFinalize) { + writeTerminalBreadcrumb(this.cwd, staged.finalSessionFile); + parentArtifacts.finalizeLastAttemptCommit(staged.attemptId); + } + } catch (error) { + const cleanupErrors: Error[] = []; + // A managed move can commit and still report failure when native code cannot + // prove durability or terminal identity. Treating that as unpublished would + // roll the artifacts back underneath a transcript that is already visible at + // the destination, so probe the destination first and preserve every owned + // artifact when the mutation may have landed. + if (!published && staged.managedParentStore && !mayCleanManagedTreeStaging(error)) { + const probeErrors: Error[] = []; + let destination: ManagedFileSnapshot | null = null; + try { + destination = staged.managedParentStore.readExpected(path.basename(staged.finalSessionFile)); + } catch (probeError) { + probeErrors.push(toError(probeError)); + } + // Absence must be proven; an unreadable destination stays fail-closed. + if (destination || probeErrors.length > 0) { + staged.publishedFinalSnapshot = destination ?? undefined; + // Latch the uncertainty so no later compensation reclaims what a possibly + // published transcript references. + staged.preservedUnproven = true; + throw new AggregateError( + [toError(error), ...probeErrors], + "Staged publication may have committed without proof; artifacts and staging were preserved for recovery.", + ); + } + } + if (published) { + try { + if (staged.managedParentStore) { + const finalSnapshot = staged.managedParentStore.readExpected(path.basename(staged.finalSessionFile)); + if (finalSnapshot) + staged.managedParentStore.removeExpected(path.basename(staged.finalSessionFile), finalSnapshot); + } else await fs.promises.rm(staged.finalSessionFile, { force: true }); + } catch (cleanupError) { + cleanupErrors.push(toError(cleanupError)); + } + } + try { + await parentArtifacts.rollbackLastAttemptCommit(staged.attemptId); + } catch (cleanupError) { + cleanupErrors.push(toError(cleanupError)); + } + try { + await this.discardStaged(); + } catch (cleanupError) { + cleanupErrors.push(toError(cleanupError)); + } + if (cleanupErrors.length > 0) + throw new AggregateError([toError(error), ...cleanupErrors], "Staged publication and cleanup both failed."); + throw error; + } + } + + /** Finalize a staged publication whose post-fence publisher completed successfully. */ + finalizeStagedCommit(): void { + const staged = this.#stagedPublication; + if (!staged?.committed || !staged.deferArtifactFinalize) return; + this.#stagedCommitArtifactParent?.finalizeLastAttemptCommit(staged.attemptId); + writeTerminalBreadcrumb(this.cwd, staged.finalSessionFile); + staged.deferArtifactFinalize = false; + } + + /** Roll back a staged publication when post-fence visibility setup fails. */ + async rollbackCommittedStaged(): Promise { + const staged = this.#stagedPublication; + if (!staged?.committed) return; + if (staged.managedParentStore) { + const current = staged.managedParentStore.readExpected(path.basename(staged.finalSessionFile)); + if ( + current && + staged.publishedFinalSnapshot && + current.identity.dev === staged.publishedFinalSnapshot.identity.dev && + current.identity.ino === staged.publishedFinalSnapshot.identity.ino + ) { + if (this.#stagedCommitArtifactParent) { + const removed = await this.#stagedCommitArtifactParent.removeNamedBestEffort( + path.basename(staged.finalSessionFile), + ); + if (!removed) throw new Error("staged_final_cleanup_failed"); + } else staged.managedParentStore.removeExpected(path.basename(staged.finalSessionFile), current); + } + } else if (staged.publishedFinalBytes) { + const current = await fs.promises.readFile(staged.finalSessionFile).catch(() => undefined); + if (current?.equals(staged.publishedFinalBytes)) + await fs.promises.rm(staged.finalSessionFile, { force: true }); + } + await this.#stagedCommitArtifactParent?.rollbackLastAttemptCommit(staged.attemptId); + staged.committed = false; + staged.discarded = true; + staged.deferArtifactFinalize = false; + } + + /** Refresh the owned final snapshot after post-fence session metadata is appended. */ + async refreshStagedCommitSnapshot(): Promise { + const staged = this.#stagedPublication; + if (!staged?.committed) return; + if (staged.managedParentStore) { + staged.publishedFinalSnapshot = + staged.managedParentStore.readExpected(path.basename(staged.finalSessionFile)) ?? undefined; + if (!staged.publishedFinalSnapshot) throw new Error("staged_session_publish_missing"); + } else staged.publishedFinalBytes = await fs.promises.readFile(staged.finalSessionFile); + } + + /** Idempotently remove an unpublished staged transcript and its owned artifacts. */ + async discardStaged(): Promise { + const staged = this.#stagedPublication; + if (!staged || staged.committed || staged.discarded) return; + // An unproven publish may already be visible at the destination; reclaiming its + // staging or artifacts would strand that transcript with dangling references. + if (staged.preservedUnproven) return; + const cleanupErrors: Error[] = []; + // Same strict single-root invariant as commitStaged: an absent or foreign adopted + // manager means the publication's own attempt root was replaced or released. + if (this.#adoptedArtifactManager?.getAttemptId() !== staged.attemptId) { + cleanupErrors.push(new Error("Staged session artifact root does not match the staged attempt.")); + } + const captureCleanupError = (error: unknown): void => { + const normalized = toError(error); + if (normalized.message !== "not_found" && !isAuthorizedPendingCleanup(normalized)) + cleanupErrors.push(normalized); + }; + try { + await this.#closePersistWriter(); + } catch (error) { + captureCleanupError(error); + } + let managedStagingBefore: ReturnType | undefined; + if (staged.managedParentStore) { + try { + managedStagingBefore = staged.managedParentStore.captureTree(SESSION_STAGING_DIRNAME); + } catch (error) { + captureCleanupError(error); + } + } + + const stagedManager = this.#adoptedArtifactManager ?? this.#artifactManager; + if (stagedManager?.getAttemptId() === staged.attemptId) { + try { + await stagedManager.discardAttemptStaging(); + } catch (error) { + captureCleanupError(error); + } + } + if (staged.managedParentStore) { + const stagedName = path.basename(staged.stagedSessionFile); + try { + const stagingStore = staged.managedStagingStore; + if (stagingStore) { + const expected = stagingStore.readExpected(stagedName); + if (expected) stagingStore.removeExpected(stagedName, expected); + } + } catch (error) { + captureCleanupError(error); + } + if (managedStagingBefore) { + try { + const after = staged.managedParentStore.captureTree(SESSION_STAGING_DIRNAME); + const beforePaths = new Set(managedStagingBefore.entries.map(entry => entry.relativePath)); + for (const entry of after.entries) { + if ( + entry.relativePath.length === 0 || + beforePaths.has(entry.relativePath) || + (!/^\.gjc-/u.test(path.posix.basename(entry.relativePath)) && + !/\.removing$/u.test(path.posix.basename(entry.relativePath))) + ) + continue; + try { + await fs.promises.rm( + path.join(staged.managedParentStore.dir, SESSION_STAGING_DIRNAME, entry.relativePath), + { recursive: entry.kind === "directory", force: true }, + ); + } catch (error) { + captureCleanupError(error); + } + } + } catch (error) { + captureCleanupError(error); + } + } + } else { + try { + await fs.promises.rm(staged.stagedSessionFile, { force: true }); + } catch (error) { + captureCleanupError(error); + } + } + if (cleanupErrors.length > 0) throw new AggregateError(cleanupErrors, "Staged session cleanup failed."); + staged.discarded = true; + } + async commitStagedNestedManaged(): Promise { + return this.commitStaged(); + } + + async discardStagedNestedManaged(): Promise { + return this.discardStaged(); + } + static async open( filePath: string, destinationInput?: SessionDestinationInput, @@ -19180,6 +19705,7 @@ export class SessionManager { migrationPolicy: SessionDirectoryMigrationPolicy = "copy-retain", sessionMemoryMode: SessionMemoryMode = "shadow", ): Promise { + if (isStagedSessionPath(filePath)) throw new Error("Staged session paths are not resumable"); const destination = destinationInput === undefined ? explicitDestination(path.dirname(filePath)) @@ -19668,7 +20194,7 @@ export class SessionManager { const listing = listManagedCandidates(resolved.scope); if (listing.kind === "error") return []; const managed = await collectSessionsFromFiles( - listing.owned.map(candidate => candidate.path), + listing.owned.filter(candidate => !isStagedSessionPath(candidate.path)).map(candidate => candidate.path), storage, ); return mergeSessionInventories(managed, await collectProjectSessions(cwd, storage)); @@ -19686,7 +20212,10 @@ export class SessionManager { ): Promise { if (!sessionDir) return await SessionManager.listManagedForResumePickerReadOnly(cwd, undefined, storage); try { - return await collectSessionsFromFiles(storage.listFilesSync(sessionDir, "*.jsonl"), storage); + return await collectSessionsFromFiles( + storage.listFilesSync(sessionDir, "*.jsonl").filter(file => !isStagedSessionPath(file)), + storage, + ); } catch { return []; } @@ -20406,7 +20935,13 @@ export class SessionManager { await manager.close(); return ownershipInspection; } - manager.#sanitizeLoadedOpenAIResponsesReplayMetadata(); + try { + await manager.#sanitizeLoadedOpenAIResponsesReplayMetadataAndPersist(); + writeTerminalBreadcrumb(manager.cwd, sessionPath); + } catch (error) { + await manager.close(); + throw error; + } return { kind: "opened", manager }; } @@ -20590,7 +21125,8 @@ export class SessionManager { logger.warn("Ignored invalid managed session candidates during global listing", { count: listing.invalid.length, }); - for (const candidate of listing.owned) logicalFiles.add(candidate.path); + for (const candidate of listing.owned) + if (!isStagedSessionPath(candidate.path)) logicalFiles.add(candidate.path); } return await collectSessionsFromFiles([...logicalFiles], storage); } catch { @@ -20649,6 +21185,7 @@ export class SessionManager { const failures: StrictInventoryFailure[] = []; const candidates: StrictInventoryCandidate[] = []; for (const managedCandidate of listing.owned) { + if (isStagedSessionPath(managedCandidate.path)) continue; const candidate = inventoryReadCandidate( storage, managedCandidate.path, diff --git a/packages/coding-agent/src/session/session-staging-paths.ts b/packages/coding-agent/src/session/session-staging-paths.ts new file mode 100644 index 0000000000..6b407da635 --- /dev/null +++ b/packages/coding-agent/src/session/session-staging-paths.ts @@ -0,0 +1,11 @@ +import * as path from "node:path"; + +export const SESSION_STAGING_DIRNAME = ".staging" as const; + +/** True when a path contains the reserved session staging directory segment. */ +export function isStagedSessionPath(filePath: string): boolean { + return path + .resolve(filePath) + .split(path.sep) + .some(segment => segment === SESSION_STAGING_DIRNAME); +} diff --git a/packages/coding-agent/src/slash-commands/builtin-registry.ts b/packages/coding-agent/src/slash-commands/builtin-registry.ts index c1659ecef9..c6565dddf0 100644 --- a/packages/coding-agent/src/slash-commands/builtin-registry.ts +++ b/packages/coding-agent/src/slash-commands/builtin-registry.ts @@ -44,6 +44,7 @@ import { } from "../setup/provider-onboarding"; import { parseThinkingLevel } from "../thinking"; import { getDisplayChangelogEntries } from "../utils/changelog"; +import { buildAutoroutingStatusReport } from "./helpers/autorouting-status"; import { buildContextReportText } from "./helpers/context-report"; import { switchSessionCredentialCommand } from "./helpers/credential-switch"; import { buildFastStatusReport } from "./helpers/fast-status-report"; @@ -1117,6 +1118,82 @@ const BUILTIN_SLASH_COMMAND_REGISTRY: ReadonlyArray = [ runtime.ctx.editor.setText(""); }, }, + { + name: "routing", + description: "Show or set up sub-agent model autorouting", + acpDescription: "Show or set up sub-agent model autorouting", + inlineHint: "[on|off|status]", + acpInputHint: "[on|off|status]", + subcommands: [ + { name: "on", description: "Enable autorouting" }, + { name: "off", description: "Disable autorouting" }, + { name: "status", description: "Show effective autorouting tiers" }, + ], + allowArgs: true, + handle: async (command, runtime) => { + const arg = command.args.trim().toLowerCase(); + if (arg === "on" || arg === "off") { + try { + // Mirror SelectorController#assertSmartRoutingWritable: the non-TUI (ACP/SDK) + // dispatch path must honor the same scoped-session guard as the TUI controller, + // otherwise a --models-scoped session can toggle routing through /routing on|off. + if ((runtime.session.scopedModels?.length ?? 0) > 0) { + throw new Error("Smart-routing settings are read-only in a --models-scoped session."); + } + runtime.settings.set("task.autorouting.enabled", arg === "on"); + } catch (err) { + return usage(`Failed to change autorouting: ${errorMessage(err)}`, runtime); + } + await runtime.output( + buildAutoroutingStatusReport({ + effective: runtime.settings.getEffectiveAutorouting(), + tiers: runtime.settings.get("task.autorouting.tiers"), + provenance: runtime.settings.get("task.autorouting.provenance"), + }), + ); + + return commandConsumed(); + } + if (arg === "" || arg === "status") { + await runtime.output( + buildAutoroutingStatusReport({ + effective: runtime.settings.getEffectiveAutorouting(), + tiers: runtime.settings.get("task.autorouting.tiers"), + provenance: runtime.settings.get("task.autorouting.provenance"), + }), + ); + + return commandConsumed(); + } + return usage("Usage: /routing [on|off|status]", runtime); + }, + handleTui: async (command, runtime) => { + const arg = command.args.trim().toLowerCase(); + runtime.ctx.editor.setText(""); + if (arg === "") { + runtime.ctx.showModelSelector({ smartRoutingOnly: true }); + return; + } + if (arg === "on" || arg === "off") { + // Route through the controller so the toggle honors the same + // scoped-session and durable-config guards as the panel. + await runtime.ctx.setAutoroutingEnabled(arg === "on"); + } else if (arg !== "status") { + runtime.ctx.showStatus("Usage: /routing [on|off|status]"); + return; + } + const report = buildAutoroutingStatusReport({ + effective: runtime.ctx.settings.getEffectiveAutorouting(), + tiers: runtime.ctx.settings.get("task.autorouting.tiers"), + provenance: runtime.ctx.settings.get("task.autorouting.provenance"), + }); + runtime.ctx.chatContainer.addChild(new Spacer(1)); + runtime.ctx.chatContainer.addChild(new DynamicBorder()); + runtime.ctx.chatContainer.addChild(new Text(report, 1, 0)); + runtime.ctx.chatContainer.addChild(new DynamicBorder()); + runtime.ctx.ui.requestRender(); + }, + }, { name: "export", priority: 50, diff --git a/packages/coding-agent/src/slash-commands/helpers/autorouting-status.ts b/packages/coding-agent/src/slash-commands/helpers/autorouting-status.ts new file mode 100644 index 0000000000..d3d6928b5f --- /dev/null +++ b/packages/coding-agent/src/slash-commands/helpers/autorouting-status.ts @@ -0,0 +1,55 @@ +import { replaceTabs, truncateToWidth } from "@gajae-code/tui"; +import { + AUTOROUTING_TIERS, + type AutoroutingEffective, + type AutoroutingProvenance, + matchesRecordedTiersFingerprint, + validateAutoroutingProvenance, +} from "../../config/autorouting-contract"; +import { validateDisplayLine } from "../../modes/components/ansi-display-validator"; + +/** Longest rendered chain/diagnostic before truncation. */ +const MAX_STATUS_LINE_WIDTH = 200; + +/** + * Selectors and diagnostics originate in user-editable config, so a hand-edited + * value can carry tabs or terminal control sequences. Strip them before the + * string reaches a renderer. + */ +function displaySafe(text: string): string { + return truncateToWidth(validateDisplayLine(replaceTabs(text)), MAX_STATUS_LINE_WIDTH); +} + +export type AutoroutingStatusSnapshot = { + effective: AutoroutingEffective; + tiers: unknown; + provenance: AutoroutingProvenance | undefined; +}; + +/** Render settings-derived autorouting state without consulting registry/auth. */ +export function buildAutoroutingStatusReport(snapshot: AutoroutingStatusSnapshot): string { + const { effective, tiers, provenance } = snapshot; + if (!effective.active) { + const detail = + effective.issue?.detail ?? "Autorouting is disabled; every Task item uses manual model resolution."; + return `Autorouting: off\n${displaySafe(detail)}`; + } + const provenanceIssues = provenance === undefined ? [] : validateAutoroutingProvenance(provenance); + const malformed = provenanceIssues.length > 0; + const generated = !malformed && provenance !== undefined && matchesRecordedTiersFingerprint(provenance, tiers); + const label = malformed + ? "hand-authored tiers" + : generated + ? "generated" + : provenance === undefined + ? "hand-authored tiers" + : "generated, hand-edited"; + const lines = [`Autorouting: on (${label})`]; + if (malformed) lines.push("Recorded generation provenance is invalid; treating tiers as hand-authored."); + for (const tier of AUTOROUTING_TIERS) { + const chain = effective.map[tier]; + const rendered = chain && chain.length > 0 ? displaySafe(chain.join(" -> ")) : "(unmapped, falls back to manual)"; + lines.push(` ${tier}: ${rendered}`); + } + return lines.join("\n"); +} diff --git a/packages/coding-agent/src/task/executor.ts b/packages/coding-agent/src/task/executor.ts index 9bc06ce0ed..933f77b1d0 100644 --- a/packages/coding-agent/src/task/executor.ts +++ b/packages/coding-agent/src/task/executor.ts @@ -23,10 +23,16 @@ import { modelSupportsServiceTier, type ServiceTier, } from "@gajae-code/ai/core"; +import { + classifyFallbackTrigger, + type TransportFailureFacts, + transportFailureFacts, +} from "@gajae-code/ai/utils/fallback-transport"; import { type JsonSchemaValidationIssue, validateJsonSchemaValue } from "@gajae-code/ai/utils/schema"; import * as canonicalSdk from "@gajae-code/coding-agent/sdk"; import { logger, prompt, untilAborted } from "@gajae-code/utils"; import { AsyncJobManager } from "../async"; +import { AUTOROUTING_SELECTOR_MAX_LENGTH, type AutoroutingReasonCode } from "../config/autorouting-contract"; import { ModelRegistry } from "../config/model-registry"; import { formatModelString, resolveModelOverrideWithAuthFallback } from "../config/model-resolver"; import type { PromptTemplate } from "../config/prompt-templates"; @@ -45,7 +51,7 @@ import submitReminderTemplate from "../prompts/system/subagent-yield-reminder.md import { AgentRegistry } from "../registry/agent-registry"; import { discoverAuthStorage } from "../sdk"; import type { AgentSession, AgentSessionEvent, ForkContextSeed } from "../session/agent-session"; -import type { ArtifactManager } from "../session/artifacts"; +import { ArtifactManager } from "../session/artifacts"; import type { AuthStorage } from "../session/auth-storage"; import { SKILL_PROMPT_MESSAGE_TYPE } from "../session/messages"; import { SessionManager, type SessionMemoryMode } from "../session/session-manager"; @@ -67,6 +73,10 @@ import { persistTaskTokenLog, taskTokenLogFromUsage } from "./token-log"; import { type AgentDefinition, type AgentProgress, + type AutoroutingAttempt, + type AutoroutingAttemptCode, + type AutoroutingPreflightFailure, + assertRoutingEvidenceInvariant, createLocalErrorSummary, createSetupFailureSummary, hasCompleteUsageCostBreakdown, @@ -81,8 +91,12 @@ import { TASK_SUBAGENT_EVENT_CHANNEL, TASK_SUBAGENT_LIFECYCLE_CHANNEL, TASK_SUBAGENT_PROGRESS_CHANNEL, + type TaskRoutingEvidence, type TaskToolDetails, } from "./types"; + +export type { AutoroutingPreflightFailure } from "./types"; + import { type ExecutorExecutionMode, resolveUltragoalRedTeamActivation } from "./ultragoal-redteam-activation"; /** Agent event types to forward for progress tracking. */ @@ -214,6 +228,17 @@ export interface ExecutorOptions { executionMode?: ExecutorExecutionMode; context?: string; description?: string; + routing?: TaskRoutingEvidence; + /** Ordered, normalized autorouting candidates for the cross-phase preflight ledger. */ + autoroutingCandidates?: string[]; + autoroutingSkips?: Array<{ selector: string; code: AutoroutingReasonCode }>; + autoroutingPreflightErrors?: Map; + autoroutingPreflight?: boolean; + autoroutingAttemptId?: string; + preflightProbe?: boolean; + preflightDurable?: boolean; + preflightFenceCallback?: () => void; + index: number; id: string; modelOverride?: string | string[]; @@ -329,6 +354,23 @@ export class ManagedTaskPersistence { this.#artifacts.assertManagedBinding(); return session; } + async openStagedSession(attemptId = this.#taskId): Promise { + const store = this.#artifacts.getManagedStore(); + if (!store) throw new Error("Managed task persistence authority is unavailable"); + this.#artifacts.assertManagedBinding(); + const sessionFile = path.join(this.#artifacts.dir, `${this.#taskId}.jsonl`); + const session = await SessionManager.stagedNestedManaged( + sessionFile, + SessionManager.nestedManagedDestination(store, this.#artifacts.dir), + store, + undefined, + attemptId, + ); + const stagedArtifacts = this.#artifacts.createAttemptStaging(attemptId); + session.adoptArtifactManager(stagedArtifacts, this.#artifacts); + this.#artifacts.assertManagedBinding(); + return session; + } async publishOutput(rawOutput: string, metadata: Uint8Array): Promise { await withArtifactManagerFinalizationTurn(this.#artifacts, () => @@ -818,10 +860,134 @@ export function createSubagentSettings(baseSettings: Settings, inheritedServiceT }); } +/** + * Finalize routing evidence at the executor return boundary: the effective + * model is the terminal provider-reported model when present, otherwise the + * auth-resolved model; substitution causes are appended in order. + */ +export function finalizeRoutingEvidence( + routing: TaskRoutingEvidence | undefined, + state: { + resolvedModelString: string | undefined; + lastAssistantModelString: string | undefined; + authFallbackUsed: boolean; + assistantModelMismatch: boolean; + }, +): TaskRoutingEvidence | undefined { + if (!routing) return undefined; + const effectiveModel = state.lastAssistantModelString ?? state.resolvedModelString; + const substitutions: TaskRoutingEvidence["substitutions"] = []; + if (state.authFallbackUsed) substitutions.push("auth_substituted"); + if (state.assistantModelMismatch) substitutions.push("assistant_model_mismatch"); + if (!effectiveModel) { + if (!routing.notExecuted && !routing.terminal && !routing.attempts) return undefined; + const terminalEvidence = { ...routing, substitutions }; + assertRoutingEvidenceInvariant(terminalEvidence); + return terminalEvidence; + } + const evidence = { + ...routing, + effectiveModel, + ...(state.resolvedModelString && state.resolvedModelString !== effectiveModel + ? { authResolvedModel: state.resolvedModelString } + : {}), + substitutions, + }; + assertRoutingEvidenceInvariant(evidence); + return evidence; +} + /** * Run a single agent in-process. */ -export async function runSubprocess(options: ExecutorOptions): Promise { +class AutoroutingProbeAcceptedError extends Error { + readonly code = "autorouting_probe_accepted"; + constructor() { + super("autorouting_probe_accepted"); + } +} + +function toError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + +function formatExecutionError(error: unknown): string { + if (error instanceof AggregateError) { + const causes = error.errors.map(cause => (cause instanceof Error ? cause.message : String(cause))).join("; "); + return causes ? `${error.message}: ${causes}` : error.message; + } + return error instanceof Error ? error.stack || error.message : String(error); +} + +function transportFactsFromError(error: unknown): TransportFailureFacts | undefined { + if (!error || typeof error !== "object") return undefined; + const value = error as { transportFailure?: unknown }; + if (!value.transportFailure || typeof value.transportFailure !== "object") return undefined; + return value.transportFailure as TransportFailureFacts; +} + +/** + * True only for the explicit "no credential found for this candidate" signal thrown at the + * auth_resolve preflight step, never for an unexpected exception the credential lookup itself + * raised (keychain access denied, corrupted store, I/O failure). auth_resolve is the one op + * that always advances to the next candidate regardless of any transient marker, so it must + * only ever be reached via this explicit, typed signal — not by inferring intent from which + * preflight phase happened to be executing when an unrelated error was thrown. + */ +function isCredentialMissingSignal(error: unknown): boolean { + return Boolean( + error && typeof error === "object" && (error as { credentialMissing?: unknown }).credentialMissing === true, + ); +} + +export function classifyAutoroutingPreflightFailure( + error: unknown, + op: Extract["op"], +): AutoroutingPreflightFailure { + const facts = transportFactsFromError(error) ?? transportFailureFacts(error); + if (facts) return { kind: "transport", class: classifyFallbackTrigger(facts).class }; + const typedTransient = + error && typeof error === "object" && typeof (error as { transient?: unknown }).transient === "boolean" + ? (error as { transient: boolean }).transient + : undefined; + // An exception raised while op is auth_resolve is only the deliberate missing-credential + // signal when it carries the explicit marker; any other error surfacing during that window + // (an unexpected keychain/config failure) must fail closed like every other unclassified + // local error, not silently advance as if credentials were simply absent. + const resolvedOp = op === "auth_resolve" && !isCredentialMissingSignal(error) ? "preflight_validation" : op; + return { + kind: "local", + op: resolvedOp, + // Local setup failures are fail-closed: only an explicit transient marker + // can advance a candidate. Unknown session/tool errors are terminal. + transient: typedTransient === true, + }; +} + +/** + * Map a preflight failure to the attempt code that is recorded in routing evidence AND whether the + * ledger may advance to the next candidate. These are two different questions: a + * `config_invalid_terminal` / `unclassified_terminal` code is recorded *and* stops the ledger, so + * the caller must never infer "advance" from the mere presence of a code. + */ +function autoroutingAttemptDisposition(failure: AutoroutingPreflightFailure): { + code: AutoroutingAttemptCode; + advance: boolean; +} { + if (failure.kind === "local" && failure.op === "auth_resolve") + return { code: "credential_unavailable", advance: true }; + if ( + failure.kind === "local" && + (failure.op === "session_open" || failure.op === "tool_bootstrap") && + failure.transient + ) + return { code: "spawn_transient_retry", advance: true }; + if (failure.kind === "local" && (!failure.transient || failure.op === "preflight_validation")) + return { code: "config_invalid_terminal", advance: false }; + return { code: "unclassified_terminal", advance: false }; +} + +export async function runSubprocessOnce(options: ExecutorOptions): Promise { const { cwd, agent, @@ -959,6 +1125,8 @@ export async function runSubprocess(options: ExecutorOptions): Promise(); let llmRequestStarted = false; + let preflightFenceCrossed = false; + let preflightFailure: AutoroutingPreflightFailure | undefined; + let preflightCommitFailure = false; + let lifecycleStarted = false; + let liveHandleRegistered = false; + let probeAccepted = false; + let preflightOperation: Extract["op"] = "preflight_validation"; const seenAssistantMessageIdentities = new Set(); // Accumulate usage incrementally from message_end events (no memory for streaming events) @@ -1395,6 +1570,8 @@ export async function runSubprocess(options: ExecutorOptions): Promise => { + let openedSessionManager: SessionManager | null = null; + let liveHandleManager: AsyncJobManager | undefined; + const liveSubagentId = options.subagentId ?? id; const sessionAbortController = new AbortController(); let exitCode = 0; let error: string | undefined; @@ -1537,11 +1721,12 @@ export async function runSubprocess(options: ExecutorOptions): Promise { + if (options.preflightProbe || liveHandleRegistered || !manager) return; manager.registerLiveHandle(liveSubagentId, { requestPause: () => { pauseRequested = true; @@ -1803,10 +2042,10 @@ export async function runSubprocess(options: ExecutorOptions): Promise { + if (options.preflightProbe || lifecycleStarted || !options.eventBus) return; options.eventBus.emit(TASK_SUBAGENT_LIFECYCLE_CHANNEL, { id, agent: agent.name, @@ -1816,6 +2055,11 @@ export async function runSubprocess(options: ExecutorOptions): Promise { + let postFencePublication: Promise | undefined; + const publishPostFence = async (): Promise => { + if (!options.preflightDurable) return; + if (postFencePublication) return postFencePublication; + postFencePublication = (async () => { + if (!openedSessionManager) throw new Error("preflight session manager unavailable"); + try { + // The staged transcript and artifacts become visible only here, at + // the real provider acceptance fence. + await openedSessionManager.commitStaged({ deferArtifactFinalize: true }); + session.sessionManager.appendSessionInit({ + systemPrompt: session.agent.state.systemPrompt.join("\n\n"), + task, + tools: session.getActiveToolNames(), + outputSchema, + forkContext: options.forkContextSeed?.metadata, + }); + await openedSessionManager.refreshStagedCommitSnapshot(); + registerLiveHandle(); + emitLifecycleStart(); + openedSessionManager.finalizeStagedCommit(); + } catch (error) { + preflightCommitFailure = true; + try { + await openedSessionManager.rollbackCommittedStaged(); + } catch (cleanupError) { + throw new AggregateError( + [toError(error), toError(cleanupError)], + "Post-fence publication and cleanup both failed.", + ); + } + throw error; + } + })(); + return postFencePublication; + }; + const markLlmRequestStarted = async () => { + if (options.preflightProbe) throw new AutoroutingProbeAcceptedError(); + if (options.preflightDurable) await publishPostFence(); llmRequestStarted = true; + preflightFenceCrossed = true; + options.preflightFenceCallback?.(); }; const promptOptions = { attribution: "agent" as const, @@ -2127,6 +2414,14 @@ export async function runSubprocess(options: ExecutorOptions): Promise"; +} + +export function buildBoundedRoutingSkips( + skips: ExecutorOptions["autoroutingSkips"], +): Pick { + if (!skips || skips.length === 0) return {}; + const retained = skips.slice(0, 16).map(skip => ({ selector: boundedSelector(skip.selector), code: skip.code })); + const omitted = skips.slice(16); + const omittedByCode: NonNullable = {}; + for (const skip of omitted) omittedByCode[skip.code] = (omittedByCode[skip.code] ?? 0) + 1; + return { + skips: retained, + ...(omitted.length > 0 ? { omittedSkipCount: omitted.length, omittedByCode } : {}), + }; +} + +function evidenceWithAttempts( + routing: TaskRoutingEvidence | undefined, + attempts: readonly AutoroutingAttempt[], + terminal?: TaskRoutingEvidence["terminal"], +): TaskRoutingEvidence | undefined { + if (!routing) return undefined; + const evidence: TaskRoutingEvidence = { + ...routing, + requestedSelector: boundedSelector(routing.requestedSelector), + attempts: attempts.map(attempt => ({ ...attempt, selector: boundedSelector(attempt.selector) })), + ...(terminal ? { terminal, notExecuted: true, effectiveModel: undefined } : {}), + }; + assertRoutingEvidenceInvariant(evidence); + return evidence; +} + +function preflightTerminalResult( + options: ExecutorOptions, + attempts: readonly AutoroutingAttempt[], + terminal: "preflight_exhausted" | "all_candidates_skipped", + prior?: SingleResult, +): SingleResult { + const result: SingleResult = prior + ? { ...prior } + : { + index: options.index, + id: options.id, + agent: options.agent.name, + agentSource: options.agent.source, + task: options.task, + assignment: options.assignment, + description: options.description, + exitCode: 1, + output: "", + stderr: "Autorouting preflight exhausted.", + truncated: false, + durationMs: 0, + tokens: 0, + }; + // Preserve a diagnostic from the last attempt so terminalization does not erase why the candidate + // failed (including an appended pre-fence cleanup failure). It MUST go through + // createSetupFailureSummary, the established egress sanitizer: it redacts authorization/cookie + // headers, URL credentials, API-key labels, bare provider tokens and local/Windows absolute paths, + // collapses all whitespace (so CR/LF/TAB cannot survive for newline injection), and caps length. + // Never interpolate a raw `error` string here — it may carry a full stack with secrets or paths. + const rawDiagnostic = + typeof prior?.error === "string" && prior.error.trim().length > 0 + ? prior.error + : (prior?.setupFailure?.summary ?? ""); + const sanitizedDiagnostic = rawDiagnostic.trim() ? createSetupFailureSummary(rawDiagnostic).summary : ""; + const priorDiagnostic = + sanitizedDiagnostic && sanitizedDiagnostic !== "Subagent setup failed." ? sanitizedDiagnostic : ""; + result.exitCode = 1; + result.output = ""; + result.stderr = priorDiagnostic + ? `Autorouting preflight exhausted. Last candidate diagnostic: ${priorDiagnostic}` + : "Autorouting preflight exhausted."; + result.error = result.stderr; + result.setupFailure = { + summary: priorDiagnostic + ? `Autorouting preflight did not accept a candidate. Last candidate diagnostic: ${priorDiagnostic}` + : "Autorouting preflight did not accept a candidate.", + }; + result.routing = evidenceWithAttempts(options.routing, attempts, terminal); + return result; +} + +/** Run routed initial tasks through the bounded probe/durable candidate ledger. */ +export async function runSubprocess(options: ExecutorOptions): Promise { + if ( + !options.autoroutingPreflight || + (options.runMode ?? "initial") !== "initial" || + !options.autoroutingCandidates + ) { + return runSubprocessOnce(options); + } + const attempts: AutoroutingAttempt[] = []; + const consumed = new Set(); + const skips = buildBoundedRoutingSkips(options.autoroutingSkips); + // Candidates were already validated and pinned against the live model snapshot by + // normalizeTierSelector before reaching here. boundedSelector is an evidence/telemetry + // sanitizer (NFKC normalization, control-char stripping, 256-char truncation) meant for + // text that gets rendered or persisted; running it on the live selector could compose + // characters differently or truncate a long-but-valid selector, sending execution to a + // model that never passed preflight. Only bound the copies that reach evidence/telemetry + // (attempts, skips, requestedSelector), never the selector actually used for modelOverride. + const candidates = options.autoroutingCandidates.filter(selector => selector.length > 0); + const durablePublicationAvailable = + options.managedPersistence !== undefined || + (typeof options.sessionFile === "string" && options.sessionFile.length > 0); + const routedOptions = options.routing ? { ...options.routing, ...skips } : options.routing; + if (candidates.length === 0) + return preflightTerminalResult({ ...options, routing: routedOptions }, attempts, "all_candidates_skipped"); + let prior: SingleResult | undefined; + for (const selector of candidates) { + if (consumed.has(selector) || consumed.size >= 3) continue; + consumed.add(selector); + if (options.autoroutingPreflightErrors?.has(selector)) { + const preflightError = options.autoroutingPreflightErrors.get(selector); + const failure = classifyAutoroutingPreflightFailure(preflightError, "auth_resolve"); + const { code } = autoroutingAttemptDisposition(failure); + attempts.push({ selector, phase: "probe", code }); + return preflightTerminalResult({ ...options, routing: routedOptions }, attempts, "preflight_exhausted"); + } + const probe = await runSubprocessOnce({ + ...options, + autoroutingPreflight: false, + preflightProbe: true, + preflightDurable: false, + modelOverride: [selector], + parentActiveModelPattern: undefined, + autoroutingCandidates: undefined, + autoroutingSkips: undefined, + sessionFile: null, + artifactsDir: undefined, + persistArtifacts: false, + managedPersistence: undefined, + parentArtifactManager: undefined, + routing: undefined, + }); + prior = probe; + if (!probe.preflightProbeAccepted) { + const failure = probe.preflightFailure ?? { kind: "local", op: "preflight_validation", transient: false }; + const { code, advance } = autoroutingAttemptDisposition(failure); + attempts.push({ selector, phase: "probe", code }); + if (!advance) + return preflightTerminalResult( + { ...options, routing: routedOptions }, + attempts, + "preflight_exhausted", + probe, + ); + continue; + } + attempts.push({ selector, phase: "probe", code: "probe_passed" }); + const durable = await runSubprocessOnce({ + ...options, + autoroutingPreflight: false, + preflightProbe: false, + // A synchronous Task may have no child transcript or artifact authority. In + // that case there is nothing durable to publish at the provider fence, so + // execute the accepted candidate through the artifact-only path instead of + // asking an in-memory SessionManager to commit a nonexistent staged session. + preflightDurable: durablePublicationAvailable, + autoroutingAttemptId: `${options.id}-${consumed.size}`, + modelOverride: [selector], + parentActiveModelPattern: undefined, + autoroutingCandidates: undefined, + autoroutingSkips: undefined, + routing: undefined, + }); + prior = durable; + if (durable.preflightCommitFailure) { + attempts.push({ selector, phase: "durable", code: "post_acceptance_failure" }); + return { ...durable, routing: evidenceWithAttempts(routedOptions, attempts) }; + } + if (durable.preflightFenceCrossed) { + if (durable.exitCode === 0) { + attempts.push({ selector, phase: "durable", code: "accepted" }); + return { ...durable, routing: evidenceWithAttempts(routedOptions, attempts) }; + } + attempts.push({ selector, phase: "durable", code: "post_acceptance_failure" }); + return { ...durable, routing: evidenceWithAttempts(routedOptions, attempts) }; + } + const failure = durable.preflightFailure ?? { kind: "local", op: "preflight_validation", transient: false }; + const { code, advance } = autoroutingAttemptDisposition(failure); + attempts.push({ selector, phase: "durable", code }); + if (!advance) + return preflightTerminalResult( + { ...options, routing: routedOptions }, + attempts, + "preflight_exhausted", + durable, + ); + } + return preflightTerminalResult({ ...options, routing: routedOptions }, attempts, "preflight_exhausted", prior); +} diff --git a/packages/coding-agent/src/task/index.ts b/packages/coding-agent/src/task/index.ts index 79deed5327..259d66d3f4 100644 --- a/packages/coding-agent/src/task/index.ts +++ b/packages/coding-agent/src/task/index.ts @@ -26,6 +26,8 @@ import { } from "@gajae-code/coding-agent/async"; import { $pickenv, prompt, Snowflake } from "@gajae-code/utils"; import type { ToolSession } from ".."; +import { normalizeTierSelector, type RoutingOutcome, resolveTaskRouting } from "../config/autorouting"; +import { AUTOROUTING_SELECTOR_MAX_LENGTH, type AutoroutingReasonCode } from "../config/autorouting-contract"; import { resolveProfileBindings } from "../config/model-profiles"; import { resolveAgentModelPatterns } from "../config/model-resolver"; import type { Theme } from "../modes/theme/theme"; @@ -33,7 +35,9 @@ import planModeSubagentPrompt from "../prompts/system/plan-mode-subagent.md" wit import taskDescriptionTemplate from "../prompts/tools/task.md" with { type: "text" }; import taskSummaryTemplate from "../prompts/tools/task-summary.md" with { type: "text" }; import type { ForkContextSeed } from "../session/agent-session"; +import { splitSelectorThinkingSuffix } from "../thinking"; import { formatBytes, formatDuration } from "../tools/render-utils"; +import { escapeXmlAttribute } from "../utils/xml-escape"; import { type AgentDefinition, type AgentProgress, @@ -44,6 +48,7 @@ import { type SingleResult, type TaskItem, type TaskParams, + type TaskRoutingEvidence, type TaskToolDetails, type TaskToolSchemaInstance, } from "./types"; @@ -66,7 +71,13 @@ import { generateCommitMessage } from "../utils/commit-message-generator"; import * as git from "../utils/git"; import { loadBundledAgents } from "./agents"; import { discoverAgents, filterVisibleAgents, getAgent } from "./discovery"; -import { createManagedTaskPersistence, renderSubagentUserPrompt, runSubprocess } from "./executor"; +import { + buildBoundedRoutingSkips, + createManagedTaskPersistence, + renderSubagentUserPrompt, + runSubprocess, +} from "./executor"; + import { adviseForkContextMode } from "./fork-context-advisory"; import { FORK_CONTEXT_TOKEN_BUDGET_BY_MODE } from "./fork-context-budget"; import { getTaskIdValidationError, validateAllocatedTaskId } from "./id"; @@ -322,6 +333,7 @@ function renderDescription( simpleMode: TaskSimpleMode, ircEnabled: boolean, parentSpawns: string, + autoroutingActive: boolean, ): string { const spawningDisabled = parentSpawns === ""; let filteredAgents = filterVisibleAgents(agents); @@ -339,7 +351,7 @@ function renderDescription( filteredAgents = filteredAgents.filter(a => allowed.has(a.name)); } const { contextEnabled, customSchemaEnabled } = getTaskSimpleModeCapabilities(simpleMode); - return prompt.render(taskDescriptionTemplate, { + const description = prompt.render(taskDescriptionTemplate, { agents: filteredAgents, spawningDisabled, MAX_CONCURRENCY: maxConcurrency, @@ -352,6 +364,8 @@ function renderDescription( schemaFreeMode: simpleMode === "schema-free", independentMode: simpleMode === "independent", }); + if (!autoroutingActive) return description; + return `${description}\n\n\nChoose a tier by agent role/type, per-call complexity, and cost intent: fast for mechanical/lookup/high-volume work where cheap tokens are the point; balanced (default) for ordinary implementation/review lanes; strong for deep design, hard debugging, or high-stakes review where the cost is justified. Provider availability/auth is enforced by deterministic code and is never an input to tier choice. Omitting tier is fine and routes as balanced.\n`; } function createTaskModeError(text: string): AgentToolResult { @@ -485,6 +499,47 @@ export function resolveForkContextMaxTokens(configured: number, model: Model | u return normalizeForkContextCap(configured, fallback, Number.MAX_SAFE_INTEGER); } +const ROUTING_SUMMARY_UNSAFE_RE = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/gu; + +function sanitizeRoutingSummaryValue(value: string): string { + return value.normalize("NFKC").replace(ROUTING_SUMMARY_UNSAFE_RE, " ").slice(0, AUTOROUTING_SELECTOR_MAX_LENGTH); +} + +export function findRoutingSnapshotModel(selector: string, routingSnapshot: readonly Model[]): Model | undefined { + const slash = selector.indexOf("/"); + if (slash <= 0) return undefined; + const provider = selector.slice(0, slash).toLowerCase(); + const modelId = selector.slice(slash + 1); + const providerMatches = routingSnapshot.filter(candidate => candidate.provider.toLowerCase() === provider); + return ( + providerMatches.find(candidate => candidate.id === modelId) ?? + providerMatches.find(candidate => { + const suffix = splitSelectorThinkingSuffix(modelId); + return ( + suffix.invalidSuffix === undefined && suffix.thinkingLevel !== undefined && candidate.id === suffix.selector + ); + }) + ); +} + +/** + * Project routing evidence into the XML-attribute-safe display view consumed by + * the task-summary template. The template runtime uses noEscape, so every + * interpolated attribute value must be escaped here. + */ +export function projectRoutingForSummary( + routing: TaskRoutingEvidence | undefined, +): { tier: string; effectiveModel: string; note: string } | undefined { + if (!routing) return undefined; + return { + tier: escapeXmlAttribute(sanitizeRoutingSummaryValue(routing.tier)), + effectiveModel: escapeXmlAttribute( + sanitizeRoutingSummaryValue(routing.effectiveModel ?? routing.requestedSelector), + ), + note: escapeXmlAttribute(sanitizeRoutingSummaryValue(routing.note ?? "")), + }; +} + // ═══════════════════════════════════════════════════════════════════════════ // Tool Class // ═══════════════════════════════════════════════════════════════════════════ @@ -565,9 +620,11 @@ export class TaskTool implements AgentTool[0]): Promise { + return (this.#testRunSubprocess ?? runSubprocess)(options); + } + #getTaskSimpleMode(): TaskSimpleMode { return this.session.settings.get("task.simple"); } @@ -800,12 +861,13 @@ export class TaskTool implements AgentTool { + static async create(session: ToolSession, options?: { runSubprocess?: typeof runSubprocess }): Promise { const sessionRepositoryBinding = await captureRepositoryBinding(session.cwd, { displayPath: session.cwd }); - // Authority check before discovery: session cwd must resolve to a stable binding. await assertExecutionRootMatchesRepositoryBinding(session.cwd, sessionRepositoryBinding); const { agents } = await discoverAgents(session.cwd); - return new TaskTool(session, agents, publicRepositoryBinding(sessionRepositoryBinding)); + const tool = new TaskTool(session, agents, publicRepositoryBinding(sessionRepositoryBinding)); + tool.#testRunSubprocess = options?.runSubprocess; + return tool; } /** Create catalog metadata from bundled agents only, without ambient filesystem discovery. */ @@ -1947,6 +2009,123 @@ export class TaskTool implements AgentTool ({ ...t, id: validateAllocatedTaskId(uniqueIds[i] ?? "") })); + const effectiveAutorouting = this.session.settings.getEffectiveAutorouting(); + const registry = this.session.modelRegistry as + | { + getAvailable?: () => Model[]; + getAll?: () => Model[]; + getApiKey?: (model: Model, credentialSessionId?: string) => Promise; + } + | undefined; + const routingSnapshot = registry?.getAll?.() ?? registry?.getAvailable?.(); + const routingByIndex = new Map(); + const routingCandidatesByIndex = new Map(); + const routingSkipsByIndex = new Map>(); + for (let i = 0; i < tasksWithUniqueIds.length; i++) { + const task = tasksWithUniqueIds[i]; + const outcome = routingSnapshot + ? resolveTaskRouting({ + effectiveAutorouting, + requestedTier: task.tier, + availableModels: routingSnapshot, + }) + : effectiveAutorouting.active + ? { + kind: "manual-fallback" as const, + tier: task.tier ?? "balanced", + requestedTier: task.tier, + ...(task.tier === undefined ? { defaultTierApplied: true as const } : {}), + attemptedSelectorCount: 0, + reason: "tier_unmatched" as const, + } + : { kind: "disabled" as const }; + + routingByIndex.set(i, outcome); + if (outcome.kind === "disabled" || !routingSnapshot || !effectiveAutorouting.active) continue; + + // Enumerate every configured selector before choosing a routed or manual + // outcome. This keeps AC12 evidence complete even when every selector is + // missing, malformed, or disabled. + const configured = effectiveAutorouting.map[outcome.tier] ?? []; + const candidates: string[] = []; + const skips: Array<{ selector: string; code: AutoroutingReasonCode }> = []; + const disabledProviders = new Set( + this.session.settings.get("disabledProviders").map(provider => provider.toLowerCase()), + ); + for (const selector of configured) { + const slash = selector.indexOf("/"); + const provider = slash > 0 ? selector.slice(0, slash).toLowerCase() : ""; + // Disabled takes precedence over snapshot presence: a disabled provider + // that is also absent must remain truthfully classified as disabled. + if (disabledProviders.has(provider)) { + skips.push({ selector, code: "provider_disabled" }); + continue; + } + const normalized = normalizeTierSelector(selector, routingSnapshot); + if (!("pinned" in normalized)) { + skips.push({ + selector, + code: "rejected" in normalized ? "selector_not_provider_qualified" : "snapshot_missing", + }); + continue; + } + candidates.push(normalized.pinned); + } + routingCandidatesByIndex.set(i, candidates); + routingSkipsByIndex.set(i, skips); + } + const resolveAutoroutingCandidates = async ( + index: number, + ): Promise<{ + candidates: string[]; + skips: Array<{ selector: string; code: AutoroutingReasonCode }>; + preflightErrors: Map; + }> => { + const candidates = [...(routingCandidatesByIndex.get(index) ?? [])]; + const skips = [...(routingSkipsByIndex.get(index) ?? [])]; + const preflightErrors = new Map(); + if (!registry?.getApiKey || !routingSnapshot) return { candidates, skips, preflightErrors }; + const authenticated: string[] = []; + for (const selector of candidates) { + const model = findRoutingSnapshotModel(selector, routingSnapshot); + if (!model) { + skips.push({ selector, code: "snapshot_missing" }); + continue; + } + try { + const key = await registry.getApiKey( + model, + this.session.getCredentialSessionId?.() ?? this.session.getSessionId?.() ?? undefined, + ); + if (key) authenticated.push(selector); + else skips.push({ selector, code: "credential_unavailable" }); + } catch (error) { + // Preserve the candidate for executor preflight, which records the + // terminal lookup failure in the routing ledger. + preflightErrors.set(selector, error); + authenticated.push(selector); + } + } + return { candidates: authenticated, skips, preflightErrors }; + }; + const effectivePatterns = (index: number): string | string[] => { + const outcome = routingByIndex.get(index); + return outcome?.kind === "routed" ? [outcome.pinnedSelector] : modelOverride; + }; + const routeEvidenceForSynthetic = (outcome: RoutingOutcome | undefined): TaskRoutingEvidence | undefined => { + if (!outcome || outcome.kind === "disabled") return undefined; + return { + tier: outcome.tier, + requestedTier: outcome.requestedTier, + defaultTierApplied: outcome.defaultTierApplied, + requestedSelector: outcome.kind === "routed" ? outcome.pinnedSelector : "manual-model-chain", + notExecuted: true, + substitutions: [], + manualFallbackReason: outcome.kind === "manual-fallback" ? outcome.reason : undefined, + note: `${outcome.tier}; ${outcome.kind === "manual-fallback" ? outcome.reason : "not-executed"}`, + }; + }; + const availableSkills = [...(this.session.skills ?? [])]; // Resolve autoload skills from agent definition against available skills const resolvedAutoloadSkills = @@ -1981,7 +2160,7 @@ export class TaskTool implements AgentTool { + if (!outcome || outcome.kind === "disabled") return undefined; + const noteParts = [ + `${outcome.tier}${outcome.defaultTierApplied ? " (default)" : ""}`, + outcome.kind === "manual-fallback" ? outcome.reason : undefined, + freshOnResume ? "freshOnResume" : undefined, + ].filter((part): part is string => part !== undefined); + return { + tier: outcome.tier, + requestedTier: outcome.requestedTier, + defaultTierApplied: outcome.defaultTierApplied, + requestedSelector: outcome.kind === "routed" ? outcome.pinnedSelector : "manual-model-chain", + effectiveModel: outcome.kind === "routed" ? outcome.pinnedSelector : "manual-model-chain", + substitutions: [], + manualFallbackReason: outcome.kind === "manual-fallback" ? outcome.reason : undefined, + freshOnResume: freshOnResume ? true : undefined, + note: noteParts.join("; "), + }; + }; + + const effectiveRunMode = overrides?.runMode ?? executionOverrides?.runMode; + const autoroutingInitial = + routingOutcome?.kind === "routed" && (effectiveRunMode ?? "initial") === "initial"; + const autoroutingData = + (effectiveRunMode ?? "initial") === "initial" && routingOutcome && routingOutcome.kind !== "disabled" + ? routingOutcome.kind === "routed" + ? await resolveAutoroutingCandidates(index) + : { + candidates: undefined, + skips: [...(routingSkipsByIndex.get(index) ?? [])], + preflightErrors: new Map(), + } + : { candidates: undefined, skips: undefined, preflightErrors: new Map() }; + const routingForRun = routeEvidence( + routingOutcome, + effectiveRunMode === "resume" || effectiveRunMode === "message", + ); + if (routingForRun && autoroutingData.skips) + Object.assign(routingForRun, buildBoundedRoutingSkips(autoroutingData.skips)); const taskSessionFile = managedPersistence ? null : (overrides?.sessionFile ?? executionOverrides?.sessionFiles?.get(task.id) ?? null); @@ -2050,7 +2274,7 @@ export class TaskTool implements AgentTool { progressMap.set(index, { ...structuredClone(progress), @@ -2127,7 +2358,7 @@ export class TaskTool implements AgentTool { progressMap.set(index, { ...structuredClone(progress), @@ -2291,7 +2528,9 @@ export class TaskTool implements AgentTool receipt.roi?.lowRoi).map(receipt => receipt.id), }; } +function validatedRoutingEvidence(value: TaskRoutingEvidence | undefined): TaskRoutingEvidence | undefined { + if (!value) return undefined; + try { + const bounded: TaskRoutingEvidence = { + ...value, + requestedSelector: value.requestedSelector.slice(0, AUTOROUTING_SELECTOR_MAX_LENGTH), + skips: value.skips + ?.slice(0, 16) + .map(skip => ({ ...skip, selector: skip.selector.slice(0, AUTOROUTING_SELECTOR_MAX_LENGTH) })), + attempts: value.attempts + ?.slice(0, 6) + .map(attempt => ({ ...attempt, selector: attempt.selector.slice(0, AUTOROUTING_SELECTOR_MAX_LENGTH) })), + }; + assertRoutingEvidenceInvariant(bounded); + return bounded; + } catch { + return undefined; + } +} export function buildTaskReceipt(raw: SingleResult): TaskResultReceipt { // Receipts only include outputRef when production code kept outputMeta after a @@ -288,6 +311,7 @@ export function buildTaskReceipt(raw: SingleResult): TaskResultReceipt { contextWindow: raw.contextWindow, modelOverride: raw.modelOverride, modelSubstitutionWarning: raw.modelSubstitutionWarning, + routing: validatedRoutingEvidence(raw.routing), usage: raw.usage, cost: raw.usage?.cost.total, usageCostBreakdownComplete: diff --git a/packages/coding-agent/src/task/render.ts b/packages/coding-agent/src/task/render.ts index a55c5df25e..ec5a17edbe 100644 --- a/packages/coding-agent/src/task/render.ts +++ b/packages/coding-agent/src/task/render.ts @@ -7,7 +7,7 @@ import path from "node:path"; import type { Component } from "@gajae-code/tui"; import { Text } from "@gajae-code/tui"; -import { formatNumber, sanitizeText } from "@gajae-code/utils"; +import { formatNumber, sanitizeDisplayLine, sanitizeText } from "@gajae-code/utils"; import type { RenderResultOptions } from "../extensibility/custom-tools/types"; import type { Theme } from "../modes/theme/theme"; import { @@ -913,6 +913,16 @@ function renderAgentResult(result: TaskResultReceipt, isLast: boolean, expanded: )}`, ); } + if (result.routing) { + const model = result.routing.effectiveModel ?? "not-executed"; + const note = result.routing.note ? ` ${result.routing.note}` : ""; + // effectiveModel is provider-reported and note is free-form diagnostic text; + // receipt validation bounds neither, and an embedded newline would inject a + // row past the width cap, so flatten to a single sanitized line. + lines.push( + `${continuePrefix}${theme.fg("dim", truncateToWidth(replaceTabs(sanitizeDisplayLine(`Routing: ${model}${note}`)), 90))}`, + ); + } if (result.roi?.lowRoi) { lines.push(`${continuePrefix}${theme.fg("warning", "low ROI: produced no material contribution")}`); } diff --git a/packages/coding-agent/src/task/types.ts b/packages/coding-agent/src/task/types.ts index ea699f47f8..01e6ebfa8d 100644 --- a/packages/coding-agent/src/task/types.ts +++ b/packages/coding-agent/src/task/types.ts @@ -1,7 +1,9 @@ import type { ThinkingLevel } from "@gajae-code/agent-core"; import type { Usage } from "@gajae-code/ai/core"; +import type { FallbackTriggerClass } from "@gajae-code/ai/utils/fallback-transport"; import { $env } from "@gajae-code/utils"; import * as z from "zod/v4"; +import { AUTOROUTING_SELECTOR_MAX_LENGTH, type AutoroutingReasonCode } from "../config/autorouting-contract"; import { isValidTaskId, TASK_ID_DESCRIPTION } from "./id"; import type { TaskResultReceipt } from "./receipt"; import type { SpawnRoiReconciliation } from "./roi-reconciliation"; @@ -97,6 +99,10 @@ const createTaskItemSchema = (_contextEnabled: boolean) => id: z.string().max(48).refine(isValidTaskId, TASK_ID_DESCRIPTION).describe("filesystem-safe task identifier"), description: z.string().describe("ui label, not seen by subagent"), assignment: z.string().describe(assignmentDescription), + tier: z + .enum(["fast", "balanced", "strong"]) + .optional() + .describe("Advisory unless autorouting is enabled; omitted routes as balanced."), executionMode: z .enum(["default", "ultragoal-red-team"]) .optional() @@ -595,6 +601,159 @@ export interface TaskPersistenceResult { recoveryRef?: TaskRecoveryArtifactRef; } +export type RoutingSubstitution = "auth_substituted" | "assistant_model_mismatch"; + +/** A typed failure observed while an autorouting candidate is still before the real provider fence. */ +export type AutoroutingPreflightFailure = + | { + kind: "local"; + op: "auth_resolve" | "session_open" | "tool_bootstrap" | "preflight_validation"; + transient: boolean; + } + | { kind: "transport"; class: FallbackTriggerClass }; + +export type AutoroutingAttemptCode = + | "probe_passed" + | "accepted" + | "spawn_transient_retry" + | "credential_unavailable" + | "config_invalid_terminal" + | "post_acceptance_failure" + | "unclassified_terminal"; + +export type AutoroutingAttempt = { + selector: string; + phase: "probe" | "durable"; + code: AutoroutingAttemptCode; +}; + +export type AutoroutingSkip = { + selector: string; + code: AutoroutingReasonCode; +}; + +export interface TaskRoutingEvidence { + tier: "fast" | "balanced" | "strong"; + requestedTier?: "fast" | "balanced" | "strong"; + defaultTierApplied?: true; + requestedSelector: string; + authResolvedModel?: string; + effectiveModel?: string; + notExecuted?: true; + substitutions: RoutingSubstitution[]; + manualFallbackReason?: "tier_unmatched" | "tier_missing_in_map"; + freshOnResume?: true; + note?: string; + skips?: AutoroutingSkip[]; + omittedSkipCount?: number; + omittedByCode?: Partial>; + attempts?: AutoroutingAttempt[]; + terminal?: "preflight_exhausted" | "all_candidates_skipped"; +} + +const AUTOROUTING_ATTEMPT_CODES = new Set([ + "probe_passed", + "accepted", + "spawn_transient_retry", + "credential_unavailable", + "config_invalid_terminal", + "post_acceptance_failure", + "unclassified_terminal", +]); + +const ROUTING_UNSAFE_TEXT_RE = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/u; + +function validBoundedSelector(value: unknown): value is string { + return ( + typeof value === "string" && + value.length > 0 && + value.length <= AUTOROUTING_SELECTOR_MAX_LENGTH && + !ROUTING_UNSAFE_TEXT_RE.test(value) + ); +} + +export function assertRoutingEvidenceInvariant(evidence: TaskRoutingEvidence): void { + const terminalWithoutModel = evidence.terminal !== undefined || evidence.notExecuted === true; + if ( + !terminalWithoutModel && + (!evidence.effectiveModel || evidence.effectiveModel.length > AUTOROUTING_SELECTOR_MAX_LENGTH) + ) + throw new Error("Invalid effective routing model."); + if (!validBoundedSelector(evidence.requestedSelector)) throw new Error("Invalid requested routing selector."); + if (evidence.authResolvedModel && evidence.authResolvedModel === evidence.effectiveModel) + throw new Error("authResolvedModel must differ from effectiveModel when present."); + if (evidence.authResolvedModel !== undefined && !validBoundedSelector(evidence.authResolvedModel)) + throw new Error("Invalid auth-resolved routing model."); + + if (evidence.skips && evidence.skips.length > 16) throw new Error("Too many autorouting skips."); + if (evidence.attempts && evidence.attempts.length > 6) throw new Error("Too many autorouting attempts."); + if ( + evidence.omittedSkipCount !== undefined && + (!Number.isSafeInteger(evidence.omittedSkipCount) || evidence.omittedSkipCount < 0) + ) + throw new Error("Invalid omitted autorouting skip count."); + for (const [code, count] of Object.entries(evidence.omittedByCode ?? {})) { + if (!Number.isSafeInteger(count) || count < 0) throw new Error("Invalid omitted autorouting skip aggregate."); + if ( + !( + code in + { + tier_unmatched: true, + tier_missing_in_map: true, + config_invalid: true, + map_absent: true, + selector_not_provider_qualified: true, + auth_substituted: true, + assistant_model_mismatch: true, + provider_disabled: true, + snapshot_missing: true, + credential_unavailable: true, + preflight_spawn_failed: true, + preflight_exhausted: true, + } + ) + ) + throw new Error("Invalid omitted autorouting skip code."); + } + for (const skip of evidence.skips ?? []) { + if (!validBoundedSelector(skip.selector)) throw new Error("Invalid autorouting skip selector."); + if ( + !( + skip.code in + { + tier_unmatched: true, + tier_missing_in_map: true, + config_invalid: true, + map_absent: true, + selector_not_provider_qualified: true, + auth_substituted: true, + assistant_model_mismatch: true, + provider_disabled: true, + snapshot_missing: true, + credential_unavailable: true, + preflight_spawn_failed: true, + preflight_exhausted: true, + } + ) + ) + throw new Error("Invalid autorouting skip code."); + } + for (const attempt of evidence.attempts ?? []) { + if (!validBoundedSelector(attempt.selector) || !AUTOROUTING_ATTEMPT_CODES.has(attempt.code)) + throw new Error("Invalid autorouting attempt."); + if (attempt.code === "accepted" && attempt.phase !== "durable") + throw new Error("accepted requires durable phase."); + if (attempt.code === "probe_passed" && attempt.phase !== "probe") + throw new Error("probe_passed requires probe phase."); + if (attempt.phase === "probe" && attempt.code === "post_acceptance_failure") + throw new Error("post_acceptance_failure requires durable phase."); + if (attempt.phase === "durable" && attempt.code === "probe_passed") + throw new Error("probe_passed requires probe phase."); + } + if (evidence.terminal === undefined && evidence.omittedSkipCount && !evidence.skips) + throw new Error("Omitted skips require skip evidence."); +} + /** Result from a single agent execution */ export interface SingleResult { index: number; @@ -616,6 +775,8 @@ export interface SingleResult { contextTokens?: number; /** Model's context window in tokens, when known. */ contextWindow?: number; + routing?: TaskRoutingEvidence; + modelOverride?: string | string[]; modelSubstitutionWarning?: ModelSubstitutionWarning; /** Whether the resolved subagent model ran under the effective fast service tier. */ @@ -625,6 +786,11 @@ export interface SingleResult { localErrorSummary?: LocalErrorSummary; /** Safe diagnostic for a failure before the subagent sent its first LLM request. */ setupFailure?: SetupFailureSummary; + /** Internal typed autorouting preflight outcome; receipt sanitization omits this field. */ + preflightFailure?: AutoroutingPreflightFailure; + preflightFenceCrossed?: boolean; + preflightProbeAccepted?: boolean; + preflightCommitFailure?: boolean; aborted?: boolean; abortReason?: string; diff --git a/packages/coding-agent/src/tools/tool-catalog.generated.ts b/packages/coding-agent/src/tools/tool-catalog.generated.ts index 3d0d24bc17..9a64f7abfd 100644 --- a/packages/coding-agent/src/tools/tool-catalog.generated.ts +++ b/packages/coding-agent/src/tools/tool-catalog.generated.ts @@ -2087,6 +2087,15 @@ export const TOOL_CATALOG: Readonly> = { "type": "string", "description": "per-task instructions; self-contained" }, + "tier": { + "description": "Advisory unless autorouting is enabled; omitted routes as balanced.", + "type": "string", + "enum": [ + "fast", + "balanced", + "strong" + ] + }, "executionMode": { "description": "typed executor mode: default keeps ordinary executor behavior; ultragoal-red-team injects the Ultragoal QA/red-team prompt fragment. Prefer this over free-form assignment text (#2698).", "type": "string", diff --git a/packages/coding-agent/src/utils/xml-escape.ts b/packages/coding-agent/src/utils/xml-escape.ts new file mode 100644 index 0000000000..0f9467f1dd --- /dev/null +++ b/packages/coding-agent/src/utils/xml-escape.ts @@ -0,0 +1,14 @@ +/** Escape a string for interpolation inside a double- or single-quoted XML attribute. */ +export function escapeXmlAttribute(input: string): string { + if (!/[&<>"']/.test(input)) return input; + let output = ""; + for (const char of input) { + if (char === "&") output += "&"; + else if (char === "<") output += "<"; + else if (char === ">") output += ">"; + else if (char === '"') output += """; + else if (char === "'") output += "'"; + else output += char; + } + return output; +} diff --git a/packages/coding-agent/test/acp-autorouting-notice.test.ts b/packages/coding-agent/test/acp-autorouting-notice.test.ts new file mode 100644 index 0000000000..9d355d08ef --- /dev/null +++ b/packages/coding-agent/test/acp-autorouting-notice.test.ts @@ -0,0 +1,176 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { AgentSideConnection, SessionNotification } from "@agentclientprotocol/sdk"; +import { AUTOROUTING_INACTIVE_WARNING } from "../src/config/autorouting-contract"; +import { AcpAgent } from "../src/modes/acp/acp-agent"; +import { startFixtureBrokerWithLeaseForTest } from "../src/sdk/broker/ensure"; +import { + cleanupFixtureRoots, + createFixtureBrokerEnvironment, + createFixtureRootCleanup, + type FixtureRootCleanup, + registerFixtureRuntime, + withFixtureBrokerEnvironment, +} from "./helpers/fixture-broker-cleanup"; + +const cleanupRoots: FixtureRootCleanup[] = []; + +afterEach(async () => { + await cleanupFixtureRoots(cleanupRoots); +}); + +async function waitFor(predicate: () => boolean, label: string, timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await Bun.sleep(10); + } + throw new Error(`Timed out waiting for ${label}`); +} + +function thoughtText(update: SessionNotification): string | undefined { + const value = update.update as { sessionUpdate?: unknown; content?: unknown }; + if (value.sessionUpdate !== "agent_thought_chunk") return undefined; + const content = value.content as { type?: unknown; text?: unknown } | undefined; + return content?.type === "text" && typeof content.text === "string" ? content.text : undefined; +} + +async function runAutoroutingSession(config: string): Promise<{ + updates: SessionNotification[]; + sessionId: string; + cwd: string; + agent: AcpAgent; + close: () => Promise; +}> { + const root = await mkdtemp(path.join(tmpdir(), "gjc-acp-autorouting-notice-")); + const cwd = path.join(root, "workspace"); + const agentDir = path.join(root, "agent"); + await mkdir(path.join(cwd, ".gjc"), { recursive: true }); + await writeFile(path.join(cwd, ".gjc", "config.yml"), config); + + const environment = createFixtureBrokerEnvironment(root, agentDir); + const started = await withFixtureBrokerEnvironment(() => + startFixtureBrokerWithLeaseForTest({ agentDir, env: environment }), + ); + const cleanup = createFixtureRootCleanup(root, agentDir, started.lease); + cleanupRoots.push(cleanup); + + const updates: SessionNotification[] = []; + const controller = new AbortController(); + const closed = Promise.withResolvers(); + const agent = new AcpAgent( + { + sessionUpdate: async (update: SessionNotification) => { + updates.push(update); + }, + signal: controller.signal, + closed: closed.promise, + } as unknown as AgentSideConnection, + { agentDir }, + ); + let sessionId: string | undefined; + let closedSession = false; + const close = async (): Promise => { + if (closedSession) return; + closedSession = true; + try { + if (sessionId) await agent.closeSession({ sessionId }); + } finally { + controller.abort(); + closed.resolve(); + } + }; + registerFixtureRuntime(cleanup, { + key: "acp-agent", + requiredOwner: "runtime-and-broker", + shutdown: close, + }); + + try { + await agent.initialize({ protocolVersion: 1, clientCapabilities: {} }); + const created = await waitForSession(agent, cwd); + sessionId = created.sessionId; + return { + updates, + sessionId, + cwd, + agent, + close, + }; + } catch (error) { + await close(); + throw error; + } +} + +async function waitForSession(agent: AcpAgent, cwd: string): Promise<{ sessionId: string }> { + return await Promise.race([ + agent.newSession({ cwd, additionalDirectories: [], mcpServers: [] }), + Bun.sleep(20_000).then(() => { + throw new Error("Timed out waiting for ACP session/new"); + }), + ]); +} + +const inactiveConfig = `configSchemaVersion: 1 +task: + autorouting: + enabled: true +`; + +const activeConfig = `configSchemaVersion: 1 +task: + autorouting: + enabled: true + tiers: + balanced: + - anthropic/claude-sonnet-4 +`; + +const disabledConfig = `configSchemaVersion: 1 +task: + autorouting: + enabled: false + tiers: + balanced: + - anthropic/claude-sonnet-4 +`; + +test("AC10b: ACP newSession receives one replayed inactive-autorouting warning thought", async () => { + const { updates, sessionId, close } = await runAutoroutingSession(inactiveConfig); + try { + const warningText = `[warning:autorouting] ${AUTOROUTING_INACTIVE_WARNING}\n`; + await waitFor( + () => updates.some(update => update.sessionId === sessionId && thoughtText(update) === warningText), + "inactive autorouting warning thought", + ); + await Bun.sleep(150); + + const thoughts = updates + .filter(update => update.sessionId === sessionId) + .map(thoughtText) + .filter(Boolean); + expect(thoughts).toEqual([warningText]); + } finally { + await close(); + } +}, 30_000); + +test("AC10c: usable autorouting tiers and disabled autorouting produce no thought warning", async () => { + for (const config of [activeConfig, disabledConfig]) { + const { updates, sessionId, close } = await runAutoroutingSession(config); + try { + await Bun.sleep(250); + expect( + updates + .filter(update => update.sessionId === sessionId) + .map(thoughtText) + .filter(Boolean), + ).toEqual([]); + } finally { + await close(); + } + } +}, 30_000); diff --git a/packages/coding-agent/test/acp-builtins.test.ts b/packages/coding-agent/test/acp-builtins.test.ts index 79d5566bcd..4573f1c8aa 100644 --- a/packages/coding-agent/test/acp-builtins.test.ts +++ b/packages/coding-agent/test/acp-builtins.test.ts @@ -1710,4 +1710,17 @@ describe("wave 5 — adapters and polish", () => { expect(result).toBe(false); expect(output).toEqual([]); }); + + it("/routing on|off refuses to mutate autorouting in a --models-scoped session", async () => { + const { output, runtime } = createRuntime(); + ( + runtime.session as unknown as { scopedModels: Array<{ model: { provider: string; id: string } }> } + ).scopedModels = [{ model: { provider: "anthropic", id: "claude-haiku-4-5" } }]; + + const result = await executeAcpBuiltinSlashCommand("/routing on", runtime); + + expect(result).toEqual({ consumed: true }); + expect(output[0]).toContain("read-only in a --models-scoped session"); + expect(runtime.settings.get("task.autorouting.enabled")).not.toBe(true); + }); }); diff --git a/packages/coding-agent/test/autorouting-boundary-redteam.test.ts b/packages/coding-agent/test/autorouting-boundary-redteam.test.ts new file mode 100644 index 0000000000..c2603047f5 --- /dev/null +++ b/packages/coding-agent/test/autorouting-boundary-redteam.test.ts @@ -0,0 +1,3266 @@ +import { afterAll, describe, expect, it, vi } from "bun:test"; +import { createHash } from "node:crypto"; +import * as fsSync from "node:fs"; +import * as fs from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { Model } from "@gajae-code/ai"; +import * as native from "@gajae-code/natives"; +import { getTerminalId } from "@gajae-code/tui"; +import { getTerminalSessionsDir } from "@gajae-code/utils"; +import { checkAutoroutingTierMap } from "../scripts/check-autorouting-tier-map"; +import { AsyncJobManager } from "../src/async"; +import { resolveTaskRouting } from "../src/config/autorouting"; +import { + AUTOROUTING_SELECTOR_PATTERN, + validateAutoroutingEffective, + validateAutoroutingLocal, + validateAutoroutingSetup, +} from "../src/config/autorouting-contract"; +import { canonicalJsonBytes, generateTierChains } from "../src/config/autorouting-generator"; +import { type CuratedTierLabels, validateTierMap } from "../src/config/autorouting-tier-map"; +import { ModelRegistry } from "../src/config/model-registry"; +import { Settings } from "../src/config/settings"; +import { SmartRoutingPanelComponent } from "../src/modes/components/smart-routing-panel"; +import { SelectorController } from "../src/modes/controllers/selector-controller"; +import { getThemeByName, setThemeInstance } from "../src/modes/theme/theme"; +import * as sdkModule from "../src/sdk"; +import { ArtifactManager } from "../src/session/artifacts"; +import { AuthStorage } from "../src/session/auth-storage"; +import { ManagedSessionDescendantStore, managedDirectoryRoot } from "../src/session/internal/managed-session-storage"; +import { SessionManager } from "../src/session/session-manager"; +import { FileSessionStorage, type SessionStorage } from "../src/session/session-storage"; +import { TaskTool } from "../src/task"; +import * as discoveryModule from "../src/task/discovery"; +import { + buildBoundedRoutingSkips, + classifyAutoroutingPreflightFailure, + runSubprocess, + runSubprocessOnce, +} from "../src/task/executor"; +import type { AutoroutingPreflightFailure, SingleResult, TaskRoutingEvidence } from "../src/task/types"; +import { assertRoutingEvidenceInvariant } from "../src/task/types"; +import { splitInternalUrlSel } from "../src/tools/path-utils"; + +const rootCommand = "bun test packages/coding-agent/test/autorouting-boundary-redteam.test.ts"; + +type Verdict = "passed" | "failed"; +type CaseRecord = { + id: string; + obligation: string; + invocation: string; + observed: unknown; + verdict: Verdict; + blocker?: string; +}; + +const cases: CaseRecord[] = []; + +function record( + id: string, + obligation: string, + invocation: string, + observed: unknown, + pass: boolean, + blocker?: string, +): void { + const executedInvocation = invocation.includes(" -t ") ? rootCommand : invocation; + cases.push({ + id, + obligation, + invocation: executedInvocation, + observed, + verdict: pass ? "passed" : "failed", + ...(blocker ? { blocker } : {}), + }); +} + +function model(provider: string, id: string, reasoning = true, extra: Record = {}): Model { + return { + provider, + id, + name: id, + api: "openai-completions", + baseUrl: "https://example.invalid", + reasoning, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4096, + ...extra, + } as Model; +} + +function bytes(value: unknown): string { + return new TextDecoder().decode(canonicalJsonBytes(value)); +} + +function fingerprint(value: unknown): string { + return createHash("sha256").update(canonicalJsonBytes(value)).digest("hex"); +} + +const syntheticLabels = { + "alpha/fast": [{ tier: "fast", rank: 1 }], + "alpha/slow": [{ tier: "fast", rank: 2 }], + "alpha/strong": [{ tier: "strong", effort: "high", rank: 1 }], + "beta/fast": [{ tier: "fast", rank: 1 }], + "gamma/fast": [{ tier: "fast", rank: 1 }], +} satisfies CuratedTierLabels; +const syntheticMap = { labels: syntheticLabels, skips: {}, version: 1 }; +const syntheticCatalog = [ + model("alpha", "fast"), + model("alpha", "slow"), + model("alpha", "strong"), + model("beta", "fast"), + model("gamma", "fast"), +]; + +function testContext(catalog: readonly Model[], settings = Settings.isolated()) { + const ui = { requestRender: vi.fn(), setFocus: vi.fn() }; + const ctx = { + settings, + session: { + scopedModels: [], + modelRegistry: { + getAll: () => catalog, + getAvailable: () => catalog, + }, + }, + ui, + showStatus: vi.fn(), + showError: vi.fn(), + notifyConfigChanged: vi.fn(async () => {}), + }; + return { ctx, settings, ui }; +} + +const taskAgent = { + name: "task", + description: "General task agent", + systemPrompt: "task", + source: "bundled" as const, + model: ["manual/frontmatter"], + blocking: true, +}; + +function taskSession( + settingsOverrides: Record, + catalog: readonly Model[], + getApiKey: (entry: Model) => Promise, +): any { + return { + cwd: process.cwd(), + hasUI: false, + settings: Settings.isolated(settingsOverrides), + getSessionFile: () => null, + getSessionSpawns: () => "*", + modelRegistry: { + getAll: () => catalog, + getAvailable: () => catalog, + getApiKey, + }, + }; +} + +function successResult(options: Parameters[0]): SingleResult { + return { + index: options.index, + id: options.id, + agent: options.agent.name, + agentSource: options.agent.source, + task: options.task, + assignment: options.assignment, + description: options.description, + exitCode: 0, + output: "ok", + stderr: "", + truncated: false, + durationMs: 1, + tokens: 1, + modelOverride: options.modelOverride, + }; +} + +async function terminalBreadcrumbBytes(): Promise { + const id = getTerminalId(); + if (!id) return null; + try { + return (await readFile(path.join(getTerminalSessionsDir(), id))).toString("base64"); + } catch { + return null; + } +} + +async function tree(root: string): Promise { + const output: string[] = []; + const walk = async (directory: string, prefix: string): Promise => { + let entries: Array; + try { + entries = await readdir(directory, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { + const relative = path.join(prefix, entry.name); + if (entry.isDirectory()) await walk(path.join(directory, entry.name), relative); + else output.push(relative); + } + }; + await walk(root, ""); + return output; +} + +async function fileBytes(filePath: string): Promise { + try { + return (await readFile(filePath)).toString("base64"); + } catch { + return null; + } +} + +afterAll(async () => { + await mkdir("artifacts", { recursive: true }); + const grouped = { + algorithm: cases.filter(item => + [ + "generator-perturbations", + "eligibility-ordering", + "curation-gate", + "disabled-path-parity", + "gen2-b1-forged-provenance-race", + ].includes(item.id), + ), + api: cases.filter(item => + [ + "raw-config-byte-parity", + "panel-preview-integrity", + "atomic-batch-integrity", + "skip-truthfulness", + "hostile-selector-sanitization", + "evidence-model-bound", + "gen2-b2-skip-ordering", + "gen2-b4-whitespace-placeholder", + "gen2-b5-bounds-phase-pairs", + "gen3-direct-skip-projection", + "gen2-b1-panel-substitution", + "gen3-b6-callback-and-rapid", + "gen3-shared-skip-projection", + ].includes(item.id), + ), + staging: cases.filter(item => + [ + "retry-bounds-deny-table", + "staging-residue-and-rekey", + "attempt-id-traversal", + "gen2-b3-attempt-id-variants", + "gen2-post-fence-remap-rollback", + "gen3-b7-remap-cycle", + "gen3-breadcrumb-aggregate-interleaving", + "gen4-b8-selector-tail-parser", + "gen4-b9-remap-cycle-nested", + "gen4-c1-cleanup-fail-closed", + "gen4-c2-successful-discard-advances", + "gen4-c3-post-fence-terminal-ledger", + "gen5-disposition-and-budget", + "gen5-terminal-diagnostic-bounds", + "gen5-uri-rekey-grammar", + "gen5-fence-disposition", + "gen6-diagnostic-redaction-unicode", + "gen6-uri-whole-token-shapes", + ].includes(item.id), + ), + }; + const runnerObserved = { + casePassCount: cases.filter(item => item.verdict === "passed").length, + caseFailCount: cases.filter(item => item.verdict === "failed").length, + expectCount: cases.length, + exitCode: cases.some(item => item.verdict === "failed") ? 1 : 0, + }; + await Bun.write( + "artifacts/autorouting-boundary-algorithm-report.json", + JSON.stringify( + { + kind: "algorithm-boundary-report", + invocation: rootCommand, + surface: "generator, curation gate, ordering, selector validation", + runnerObserved, + cases: grouped.algorithm, + verdict: grouped.algorithm.some(item => item.verdict === "failed") ? "failed" : "passed", + }, + null, + 2, + ), + ); + await Bun.write( + "artifacts/autorouting-boundary-api-package-test-report.json", + JSON.stringify( + { + kind: "api-package-test-report", + invocation: rootCommand, + surface: "settings, controller, TaskTool routing evidence", + runnerObserved, + cases: grouped.api, + verdict: grouped.api.some(item => item.verdict === "failed") ? "failed" : "passed", + }, + null, + 2, + ), + ); + await Bun.write( + "artifacts/autorouting-boundary-staging-report.json", + JSON.stringify( + { + kind: "property-test-report", + invocation: rootCommand, + surface: "preflight evidence, staged sessions, artifact re-keying", + runnerObserved, + cases: grouped.staging, + verdict: grouped.staging.some(item => item.verdict === "failed") ? "failed" : "passed", + }, + null, + 2, + ), + ); + await Bun.write( + "artifacts/autorouting-boundary-generation4-report.json", + JSON.stringify( + { + kind: "property-test-report", + generation: 4, + invocation: rootCommand, + surface: "generation-4 selector-tail remap and preflight cleanup red-team", + runnerObserved, + cases, + verdict: cases.some(item => item.verdict === "failed") ? "failed" : "passed", + }, + null, + 2, + ), + ); + await Bun.write( + "artifacts/autorouting-boundary-generation5-report.json", + JSON.stringify( + { + kind: "property-test-report", + generation: 5, + invocation: rootCommand, + surface: + "generation-5 terminal disposition, bounded diagnostics, fence semantics, and URI grammar red-team", + runnerObserved, + cases, + verdict: cases.some(item => item.verdict === "failed") ? "failed" : "passed", + }, + null, + 2, + ), + ); + await Bun.write( + "artifacts/autorouting-boundary-generation6-report.json", + JSON.stringify( + { + kind: "property-test-report", + generation: 6, + invocation: rootCommand, + surface: "generation-6 varied diagnostic egress and whole-token URI re-keying red-team", + runnerObserved, + cases, + verdict: cases.some(item => item.verdict === "failed") ? "failed" : "passed", + }, + null, + 2, + ), + ); + await Bun.write( + "artifacts/autorouting-boundary-generation8-report.json", + JSON.stringify( + { + kind: "property-test-report", + generation: 8, + invocation: rootCommand, + surface: "generation-8 artifact ownership and leaked-ID rollback fault injection", + runnerObserved, + cases: cases.filter(item => item.id.startsWith("gen8-")), + verdict: cases.some(item => item.id.startsWith("gen8-") && item.verdict === "failed") ? "failed" : "passed", + }, + null, + 2, + ), + ); +}); + +describe("autorouting boundary red-team: deterministic generator and curation", () => { + it("AC1/AC11: survives key, catalog, credential, disabled-provider, and runtime-overlay perturbations", () => { + const setup = { + schema: 1 as const, + providers: ["alpha", "beta", "gamma"], + models: ["beta/fast", "alpha/slow", "alpha/fast"], + }; + const reorderedSetup = { models: [...setup.models], providers: [...setup.providers], schema: 1 as const }; + const first = generateTierChains(setup, syntheticMap, syntheticCatalog); + const second = generateTierChains(reorderedSetup, syntheticMap, [...syntheticCatalog].reverse()); + const credentialChanged = generateTierChains( + setup, + syntheticMap, + syntheticCatalog.map(entry => + model(entry.provider, entry.id, entry.reasoning, { + baseUrl: "https://credential-state.invalid", + headers: { Authorization: "Bearer secret" }, + authenticated: false, + }), + ), + ); + const disabledFlagChanged = generateTierChains( + setup, + syntheticMap, + syntheticCatalog.map(entry => model(entry.provider, entry.id, entry.reasoning, { disabled: true })), + ); + const runtimeOverlay = generateTierChains(setup, syntheticMap, [ + ...syntheticCatalog, + model("custom", "runtime-only"), + ]); + const observed = { + first, + second, + credentialChanged, + disabledFlagChanged, + runtimeOverlay, + }; + const pass = + bytes(first) === bytes(second) && + bytes(first) === bytes(credentialChanged) && + bytes(first) === bytes(disabledFlagChanged) && + bytes(first.tiers) === bytes(runtimeOverlay.tiers); + record( + "generator-perturbations", + "AC1", + `${rootCommand} -t generator-perturbations`, + observed, + pass, + "Generator output or fingerprints changed when only credential/disabled runtime state changed.", + ); + expect(pass).toBe(true); + }); + + it("AC2/AC3/AC4: filters without reordering, never duplicates unlabeled tiers, and ignores non-catalog keys", () => { + const allowlistA = generateTierChains( + { schema: 1, providers: ["alpha", "beta"], models: ["beta/fast", "alpha/slow", "alpha/fast"] }, + syntheticMap, + syntheticCatalog, + ); + const allowlistB = generateTierChains( + { schema: 1, providers: ["alpha", "beta"], models: ["alpha/fast", "beta/fast", "alpha/slow"] }, + syntheticMap, + syntheticCatalog, + ); + const thin = generateTierChains( + { schema: 1, providers: ["qianfan"] }, + { labels: { "qianfan/only": [{ tier: "fast", rank: 1 }] }, skips: {}, version: 1 }, + [model("qianfan", "only", false)], + ); + const nonCatalog = generateTierChains( + { schema: 1, providers: ["alpha"] }, + { labels: { "alpha/ghost": [{ tier: "strong", rank: 1 }] }, skips: {}, version: 1 }, + [model("alpha", "fast")], + ); + const thinRouting = resolveTaskRouting({ + effectiveAutorouting: validateAutoroutingEffective({ enabled: true, tiers: thin.tiers }), + requestedTier: "balanced", + availableModels: [model("qianfan", "only", false)], + }); + const observed = { + allowlistA: allowlistA.tiers, + allowlistB: allowlistB.tiers, + thin: thin.tiers, + thinRouting, + nonCatalog: nonCatalog.tiers, + }; + const pass = + bytes(allowlistA.tiers) === bytes(allowlistB.tiers) && + JSON.stringify(allowlistA.tiers.fast) === JSON.stringify(["alpha/fast", "alpha/slow", "beta/fast"]) && + JSON.stringify(thin.tiers) === JSON.stringify({ fast: ["qianfan/only"] }) && + thinRouting.kind === "manual-fallback" && + thinRouting.reason === "tier_missing_in_map" && + Object.keys(nonCatalog.tiers).length === 0; + record( + "eligibility-ordering", + "AC2", + `${rootCommand} -t eligibility-ordering`, + observed, + pass, + "Allowlist reordered candidates, duplicated an unlabeled tier, or admitted a non-catalog key.", + ); + expect(pass).toBe(true); + }); + + it("AC9/AC10: rejects invalid labels, effort suffixes, rank collisions, and unlabeled current models", () => { + const catalog = [model("alpha", "one"), model("alpha", "two")]; + const collision = { + labels: { + "alpha/one": [{ tier: "fast", rank: 1 }], + "alpha/two": [{ tier: "fast", rank: 1 }], + }, + skips: {}, + version: 1, + }; + const invalidLabel = { labels: { "alpha/missing": [{ tier: "fast", rank: 1 }] }, skips: {}, version: 1 }; + const invalidEffort = { + labels: { "alpha/one": [{ tier: "fast", effort: "max", rank: 1 }] }, + skips: {}, + version: 1, + }; + const collisionError = (() => { + try { + validateTierMap(collision, catalog); + return null; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + })(); + const invalidLabelError = (() => { + try { + validateTierMap(invalidLabel, catalog); + return null; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + })(); + const invalidEffortError = (() => { + try { + validateTierMap(invalidEffort, catalog); + return null; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + })(); + const gate = checkAutoroutingTierMap( + { + alpha: { + one: { provider: "alpha", id: "one", reasoning: true }, + three: { provider: "alpha", id: "three", reasoning: false }, + }, + }, + { "alpha/one": [{ tier: "fast", rank: 1 }] }, + {}, + ); + const selectorIssues = validateAutoroutingSetup({ schema: 1, providers: ["../escape"], models: ["bare-model"] }); + const hostileProviders = ["../escape", "\u0000", "π", "p".repeat(10_000)]; + const hostileProviderIssues = validateAutoroutingSetup({ schema: 1, providers: hostileProviders }); + const hostileProviderOutput = generateTierChains( + { schema: 1, providers: hostileProviders }, + syntheticMap, + syntheticCatalog, + ); + const observed = { + collisionError, + invalidLabelError, + invalidEffortError, + gate, + selectorPattern: AUTOROUTING_SELECTOR_PATTERN, + selectorIssues, + hostileProviderIssues, + hostileProviderOutput, + }; + const pass = + collisionError?.includes("collides") === true && + invalidLabelError?.includes("absent from") === true && + invalidEffortError?.includes("Unknown tier effort") === true && + gate.ok === false && + gate.report.unlabeledKeys.includes("alpha/three") && + selectorIssues.length > 0 && + Object.keys(hostileProviderOutput.tiers).length === 0; + record( + "curation-gate", + "AC9", + `${rootCommand} -t curation-gate`, + observed, + pass, + "Curation accepted a label/effort/rank violation or failed to gate an unlabeled current model.", + ); + expect(pass).toBe(true); + }); + + it("AC1: disabled mode remains byte-equivalent to the no-routing path", () => { + const snapshot = [model("alpha", "fast")]; + const disabled = validateAutoroutingEffective({ enabled: false, tiers: { fast: ["alpha/fast"] } }); + const absent = validateAutoroutingEffective(undefined); + const disabledOutcome = resolveTaskRouting({ + effectiveAutorouting: disabled, + requestedTier: "fast", + availableModels: snapshot, + }); + const absentOutcome = resolveTaskRouting({ + effectiveAutorouting: absent, + requestedTier: "fast", + availableModels: snapshot, + }); + const observed = { + disabled, + absent, + disabledOutcome, + absentOutcome, + setupDefault: undefined, + provenanceDefault: undefined, + }; + const pass = bytes(disabledOutcome) === bytes(absentOutcome) && disabledOutcome.kind === "disabled"; + record( + "disabled-path-parity", + "AC1", + `${rootCommand} -t disabled-path-parity`, + observed, + pass, + "Disabled autorouting produced a routing outcome different from the no-routing path.", + ); + expect(pass).toBe(true); + }); +}); + +describe("autorouting boundary red-team: config and panel atomicity", () => { + it("AC1: load/flush preserves hostile-but-valid untouched bytes and omits absent setup/provenance", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-byte-parity-")); + const agentDir = path.join(root, "agent"); + const cwd = path.join(root, "workspace"); + await mkdir(agentDir, { recursive: true }); + await mkdir(cwd, { recursive: true }); + const config = + "# preserve\r\nconfigSchemaVersion: 1\r\ntask:\r\n autorouting:\r\n enabled: false\r\n preset: anthropic\r\n\r\n"; + const configPath = path.join(agentDir, "config.yml"); + await writeFile(configPath, config); + const settings = await Settings.loadForScope({ cwd, agentDir }); + await settings.flush(); + const after = await Bun.file(configPath).text(); + const observed = { + beforeBytes: Buffer.byteLength(config), + afterBytes: Buffer.byteLength(after), + byteEqual: after === config, + setup: settings.get("task.autorouting.setup"), + provenance: settings.get("task.autorouting.provenance"), + }; + const pass = after === config && observed.setup === undefined && observed.provenance === undefined; + record( + "raw-config-byte-parity", + "AC1", + `${rootCommand} -t raw-config-byte-parity`, + observed, + pass, + "Untouched config bytes changed or absent optional setup/provenance serialized.", + ); + expect(pass).toBe(true); + }); + + it("AC5/AC6/AC7/AC8: detects a forged preview and verifies atomic/toggle contracts", async () => { + const catalog = [ + model("anthropic", "claude-haiku-4-5"), + model("anthropic", "claude-sonnet-5"), + model("anthropic", "claude-sonnet-4-6"), + model("anthropic", "claude-opus-5"), + ]; + const { ctx, settings } = testContext(catalog); + const controller = new SelectorController(ctx as never); + const setup = { schema: 1 as const, providers: ["anthropic"] }; + const preview = controller.previewSmartRouting(setup); + const forgedPreview = { ...preview, tiers: { fast: ["anthropic/claude-opus-5"] } }; + await controller.applySmartRouting(setup, { preview: forgedPreview }); + const applied = settings.get("task.autorouting.tiers"); + const forgedAccepted = bytes(applied) === bytes(forgedPreview.tiers); + const previewEqualsApply = bytes(applied) === bytes(preview.tiers); + const beforeToggle = { + tiers: settings.get("task.autorouting.tiers"), + setup: settings.get("task.autorouting.setup"), + provenance: settings.get("task.autorouting.provenance"), + }; + await controller.setAutoroutingEnabled(true); + const afterToggle = { + tiers: settings.get("task.autorouting.tiers"), + setup: settings.get("task.autorouting.setup"), + provenance: settings.get("task.autorouting.provenance"), + enabled: settings.get("task.autorouting.enabled"), + }; + const observed = { + expectedPreview: preview, + forgedPreview, + applied, + forgedAccepted, + previewEqualsApply, + beforeToggle, + afterToggle, + }; + const pass = !forgedAccepted && previewEqualsApply && bytes(beforeToggle.tiers) === bytes(afterToggle.tiers); + record( + "panel-preview-integrity", + "AC6", + `${rootCommand} -t panel-preview-integrity`, + observed, + pass, + "Controller accepted a forged preview with the same setup, so applied payload differed from the generated preview.", + ); + expect(pass).toBe(true); + }); + + it("AC5/AC7: rejects partial batches and preserves hand-edit guard inputs", async () => { + const setup = { schema: 1 as const, providers: ["alpha"] }; + const provenance = { + schema: 1 as const, + source: { catalogFingerprint: "a".repeat(64), mapFingerprint: "b".repeat(64), generatorVersion: 1 }, + declarationFingerprint: "c".repeat(64), + tiersFingerprint: fingerprint({ fast: ["alpha/fast"] }), + }; + const settings = Settings.isolated({ + "task.autorouting.tiers": { fast: ["before/model"] }, + "task.autorouting.setup": setup, + "task.autorouting.provenance": provenance, + }); + const before = { + tiers: settings.get("task.autorouting.tiers"), + setup: settings.get("task.autorouting.setup"), + provenance: settings.get("task.autorouting.provenance"), + }; + let error = ""; + try { + await settings.commitAtomicBatch([ + { path: "task.autorouting.tiers", op: "set", value: { fast: ["after/model"] } }, + { path: "task.autorouting.setup", op: "set", value: undefined } as never, + ]); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + const after = { + tiers: settings.get("task.autorouting.tiers"), + setup: settings.get("task.autorouting.setup"), + provenance: settings.get("task.autorouting.provenance"), + }; + const observed = { before, after, error }; + const pass = + Boolean(error) && bytes(before.tiers) === bytes(after.tiers) && bytes(before.setup) === bytes(after.setup); + record( + "atomic-batch-integrity", + "AC5", + `${rootCommand} -t atomic-batch-integrity`, + observed, + pass, + "Atomic settings batch partially applied after a later invalid patch.", + ); + expect(pass).toBe(true); + }); +}); + +describe("autorouting boundary red-team: routing evidence, retries, and residue", () => { + it("AC12: distinguishes disabled, missing snapshot, and unavailable credentials and retains overflow accounting", async () => { + const present = model("enabled", "present"); + const snapshot = [present]; + const settings = { + "task.autorouting.enabled": true, + "task.autorouting.tiers": { + fast: ["disabled/missing", "missing/absent", "enabled/present"], + }, + disabledProviders: ["disabled"], + }; + const observedSkips: Array = []; + const stub = async (options: Parameters[0]) => { + observedSkips.push({ skips: options.autoroutingSkips, candidates: options.autoroutingCandidates }); + return successResult(options); + }; + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents: [taskAgent], projectAgentsDir: null }); + AsyncJobManager.setInstance(new AsyncJobManager({ maxRunningJobs: 4, onJobComplete: async () => {} })); + const registryApiKey = async (entry: Model): Promise => + entry.provider === "enabled" ? undefined : "key"; + const tool = await TaskTool.create(taskSession(settings, snapshot, registryApiKey), { runSubprocess: stub }); + await tool.execute("skip-case", { + agent: "task", + tasks: [{ id: "one", description: "one", assignment: "run", tier: "fast" }], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + const first = observedSkips[0] as + | { skips?: Array<{ selector: string; code: string }>; candidates?: string[] } + | undefined; + const bySelector = Object.fromEntries((first?.skips ?? []).map(entry => [entry.selector, entry.code])); + const overflowRouting = await runSubprocess({ + cwd: process.cwd(), + agent: taskAgent, + task: "overflow", + assignment: "overflow", + index: 0, + id: "overflow", + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: [], + autoroutingSkips: Array.from({ length: 20 }, (_, index) => ({ + selector: `provider/model-${index}`, + code: index % 2 === 0 ? "snapshot_missing" : "credential_unavailable", + })), + routing: { + tier: "fast", + requestedSelector: "enabled/present", + substitutions: [], + }, + }); + const observed = { first, bySelector, candidates: first?.candidates, overflowRouting: overflowRouting.routing }; + const pass = + bySelector["disabled/missing"] === "provider_disabled" && + bySelector["missing/absent"] === "snapshot_missing" && + bySelector["enabled/present"] === "credential_unavailable" && + overflowRouting.routing?.skips?.length === 16 && + overflowRouting.routing?.omittedSkipCount === 4 && + overflowRouting.routing?.omittedByCode?.snapshot_missing === 2 && + overflowRouting.routing?.omittedByCode?.credential_unavailable === 2; + record( + "skip-truthfulness", + "AC12", + `${rootCommand} -t skip-truthfulness`, + observed, + pass, + "Disabled provider, missing snapshot, and unavailable credentials were conflated or skip overflow was lost.", + ); + expect(pass).toBe(true); + }); + + it("AC13: bounds attempts, rejects deny-table retries, and preserves phase/code invariants", async () => { + const failures: Array<{ name: string; failure: AutoroutingPreflightFailure; code: string }> = [ + { + name: "auth", + failure: { kind: "transport", class: "auth" }, + code: "terminal", + }, + { + name: "quota", + failure: { kind: "transport", class: "quota" }, + code: "terminal", + }, + { + name: "rate_limit", + failure: { kind: "transport", class: "rate_limit" }, + code: "terminal", + }, + { + name: "config", + failure: { kind: "local", op: "preflight_validation", transient: false }, + code: "terminal", + }, + { + name: "credential", + failure: { kind: "local", op: "auth_resolve", transient: false }, + code: "advance", + }, + { + name: "spawn", + failure: { kind: "local", op: "session_open", transient: true }, + code: "advance", + }, + ]; + const classified = failures.map(item => { + const error = + item.failure.kind === "transport" + ? { + transportFailure: { + kind: "transport" as const, + status: item.name === "auth" ? 401 : item.name === "quota" ? 402 : 429, + ...(item.name === "quota" ? { providerCode: "insufficient_quota" } : {}), + }, + } + : item.failure; + return { + name: item.name, + failure: classifyAutoroutingPreflightFailure( + error, + item.failure.kind === "local" ? item.failure.op : "session_open", + ), + code: item.code, + }; + }); + const valid: TaskRoutingEvidence = { + tier: "fast", + requestedSelector: "provider/model", + notExecuted: true, + substitutions: [], + attempts: [ + { selector: "provider/one", phase: "probe", code: "probe_passed" }, + { selector: "provider/one", phase: "durable", code: "spawn_transient_retry" }, + { selector: "provider/two", phase: "probe", code: "probe_passed" }, + { selector: "provider/two", phase: "durable", code: "accepted" }, + ], + }; + assertRoutingEvidenceInvariant(valid); + let invalidPairing = ""; + try { + assertRoutingEvidenceInvariant({ + ...valid, + attempts: [{ selector: "provider/model", phase: "probe", code: "accepted" }], + }); + } catch (error) { + invalidPairing = error instanceof Error ? error.message : String(error); + } + const duplicateAttemptLedger = [ + "provider/one", + "provider/one", + "provider/two", + "provider/three", + "provider/four", + ].filter((selector, index, all) => all.indexOf(selector) === index); + const observed = { + classified, + valid, + invalidPairing, + uniqueCandidateCount: duplicateAttemptLedger.length, + budgetedCandidates: duplicateAttemptLedger.slice(0, 3), + }; + const pass = + classified.filter(item => item.code === "terminal").every(item => item.name !== "credential") && + classified.find(item => item.name === "auth")?.failure.kind === "transport" && + Boolean(invalidPairing) && + duplicateAttemptLedger.length === 4 && + observed.budgetedCandidates.length === 3; + record( + "retry-bounds-deny-table", + "AC13", + `${rootCommand} -t retry-bounds-deny-table`, + observed, + pass, + "A deny-table transport failure advanced, an attempt pairing escaped validation, or duplicate candidates were retried.", + ); + expect(pass).toBe(true); + }); + + it("AC13: failed staged attempts leave no discovery residue and accepted artifacts are re-keyed", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-staged-boundary-")); + const cwd = path.join(root, "cwd"); + await mkdir(cwd, { recursive: true }); + const finalPath = path.join(root, "candidate.jsonl"); + const parentArtifacts = new ArtifactManager(finalPath.slice(0, -6)); + const sibling = await parentArtifacts.save("sibling", "tool"); + const beforeBreadcrumb = await terminalBreadcrumbBytes(); + const failed = await SessionManager.openStaged(finalPath, undefined, "attempt-safe"); + const stagedArtifacts = failed.getArtifactManager(); + if (!stagedArtifacts) throw new Error("staged artifact manager unavailable"); + await stagedArtifacts.save("candidate", "tool"); + failed.appendCustomEntry("hostile", { + artifactRef: "artifact://0", + agentRef: "agent://0", + adjacentNumber: "10", + decimal: "0.5", + }); + const failedBeforeDiscard = { + final: await fileBytes(finalPath), + staging: await tree(path.join(root, ".staging")), + breadcrumb: await terminalBreadcrumbBytes(), + }; + await failed.discardStaged(); + await failed.discardStaged(); + const failedAfterDiscard = { + final: await fileBytes(finalPath), + staging: await tree(path.join(root, ".staging")), + breadcrumb: await terminalBreadcrumbBytes(), + }; + const accepted = await SessionManager.openStaged(finalPath, undefined, "attempt-accepted"); + const acceptedArtifacts = accepted.getArtifactManager(); + if (!acceptedArtifacts) throw new Error("accepted artifact manager unavailable"); + const acceptedOldId = await acceptedArtifacts.save("accepted", "tool"); + accepted.appendCustomEntry("refs", { + artifactRef: `artifact://${acceptedOldId}`, + agentRef: `agent://${acceptedOldId}`, + adjacentNumber: `10${acceptedOldId}`, + decimal: `${acceptedOldId}.5`, + }); + await accepted.commitStaged(); + const finalText = await Bun.file(finalPath).text(); + const afterBreadcrumb = await terminalBreadcrumbBytes(); + const observed = { + sibling, + failedBeforeDiscard, + failedAfterDiscard, + finalText, + stagingTreeAfterAccept: await tree(path.join(root, ".staging")), + parentArtifacts: await tree(parentArtifacts.dir), + beforeBreadcrumb, + afterBreadcrumb, + acceptedOldId, + }; + const pass = + failedBeforeDiscard.final === null && + failedAfterDiscard.final === null && + failedAfterDiscard.staging.length === 0 && + failedAfterDiscard.breadcrumb === beforeBreadcrumb && + finalText.includes("artifact://1") && + finalText.includes("agent://1") && + !finalText.includes("artifact://0") && + !finalText.includes("agent://0") && + finalText.includes('"adjacentNumber":"100"') && + finalText.includes('"decimal":"0.5"'); + record( + "staging-residue-and-rekey", + "AC13", + `${rootCommand} -t staging-residue-and-rekey`, + observed, + pass, + "Failed staged attempt left durable residue, or accepted artifact references retained stale staged IDs.", + ); + expect(pass).toBe(true); + }); + + it("injection: rejects traversal attempt IDs before any outside-staging write", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-attempt-id-")); + const finalPath = path.join(root, "final.jsonl"); + const openedPaths: string[] = []; + const baseStorage = new FileSessionStorage(); + const storage = new Proxy(baseStorage, { + get(target, property, receiver) { + if (property === "openWriter") { + return (filePath: string, options?: Parameters[1]) => { + openedPaths.push(filePath); + return target.openWriter(filePath, options); + }; + } + const value = Reflect.get(target, property, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as unknown as SessionStorage; + let error = ""; + try { + await SessionManager.openStaged(finalPath, storage, "../escaped"); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + const escapedPath = path.join(root, "escaped.jsonl"); + const escapedExists = await stat(escapedPath) + .then(() => true) + .catch(() => false); + const stagingFiles = await tree(path.join(root, ".staging")); + const outsideWrites = openedPaths.filter(filePath => filePath.includes("escaped.jsonl")); + const observed = { error, escapedPath, escapedExists, stagingFiles, openedPaths, outsideWrites }; + const pass = + error.includes("Unsafe artifact attempt id") && + outsideWrites.length === 0 && + !escapedExists && + stagingFiles.length === 0; + record( + "attempt-id-traversal", + "AC13", + `${rootCommand} -t attempt-id-traversal`, + observed, + pass, + "Unvalidated attemptId escaped the staging directory before ArtifactManager rejected it.", + ); + expect(pass).toBe(true); + }); +}); + +describe("autorouting boundary red-team: malformed evidence and hostile selectors", () => { + it("fails closed on control-only selectors and hostile local config values", async () => { + const hostile = ["../escape/model", "provider/\u0000\u0001", `provider/${"x".repeat(10_000)}`, "аlpha/model"]; + const perSelectorIssues = hostile.map(selector => validateAutoroutingLocal({ tiers: { fast: [selector] } })); + const localIssues = perSelectorIssues.flat(); + // The selector grammar caps length at the routing-evidence bound and rejects + // control bytes before they can reach routing; the other hostile shapes remain + // grammar-valid and flow to the executor's sanitization paths. + const overLongIssues = perSelectorIssues[2]; + const controlIssues = perSelectorIssues[1]; + const toleratedIssues = [0, 3].flatMap(index => perSelectorIssues[index]); + let controlOnlyError = ""; + try { + await runSubprocess({ + cwd: process.cwd(), + agent: taskAgent, + task: "hostile", + assignment: "hostile", + index: 0, + id: "hostile", + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: [], + autoroutingSkips: [{ selector: "\u0000\u0001", code: "snapshot_missing" }], + routing: { + tier: "fast", + requestedSelector: "provider/model", + substitutions: [], + }, + }); + } catch (error) { + controlOnlyError = error instanceof Error ? error.message : String(error); + } + const observed = { + hostile, + localIssueCount: localIssues.length, + localIssueDetails: localIssues.map(issue => issue.detail), + controlOnlyError, + }; + const pass = + overLongIssues.length > 0 && + controlIssues.length > 0 && + toleratedIssues.length === 0 && + controlOnlyError.length === 0; + record( + "hostile-selector-sanitization", + "AC12", + `${rootCommand} -t hostile-selector-sanitization`, + observed, + pass, + "A selector sanitized to an empty string caused executor evidence construction to throw.", + ); + expect(pass).toBe(true); + }); + + it("does not accept unbounded auth-resolved model evidence", () => { + const evidence: TaskRoutingEvidence = { + tier: "fast", + requestedSelector: "provider/model", + effectiveModel: "provider/model", + authResolvedModel: `provider/${"x".repeat(10_000)}`, + substitutions: [], + }; + let error = ""; + try { + assertRoutingEvidenceInvariant(evidence); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + const observed = { authResolvedLength: evidence.authResolvedModel?.length, error }; + const pass = error.length > 0; + record( + "evidence-model-bound", + "AC12", + `${rootCommand} -t evidence-model-bound`, + observed, + pass, + "assertRoutingEvidenceInvariant accepted an unbounded authResolvedModel selector.", + ); + expect(pass).toBe(true); + }); +}); + +describe("autorouting boundary red-team generation 2 delta re-attacks", () => { + it("B1 varied: forged provenance is discarded and a catalog race cannot persist stale preview bytes", async () => { + const catalogA = [ + model("anthropic", "claude-haiku-4-5"), + model("anthropic", "claude-sonnet-5"), + model("anthropic", "claude-sonnet-4-6"), + model("anthropic", "claude-opus-5"), + ]; + const setup = { schema: 1 as const, providers: ["anthropic"] }; + const stableContext = testContext(catalogA); + const stableController = new SelectorController(stableContext.ctx as never); + const generated = stableController.previewSmartRouting(setup); + const forged = { + ...generated, + provenance: { + ...generated.provenance, + declarationFingerprint: "e".repeat(64), + tiersFingerprint: "f".repeat(64), + source: { ...generated.provenance.source, mapFingerprint: "d".repeat(64) }, + }, + }; + const appliedForged = await stableController.applySmartRouting(setup, { preview: forged }); + const persistedForged = { + tiers: stableContext.settings.get("task.autorouting.tiers"), + provenance: stableContext.settings.get("task.autorouting.provenance"), + }; + const raceContext = testContext(catalogA); + const catalogB = [model("anthropic", "claude-haiku-4-5")]; + let catalogCalls = 0; + raceContext.ctx.session.modelRegistry = { + getAll: () => { + catalogCalls += 1; + return catalogCalls === 1 ? catalogA : catalogB; + }, + getAvailable: () => (catalogCalls === 1 ? catalogA : catalogB), + }; + const raceController = new SelectorController(raceContext.ctx as never); + const stalePreview = raceController.previewSmartRouting(setup); + const racedApplied = await raceController.applySmartRouting(setup, { preview: stalePreview }); + const racedPersisted = raceContext.settings.get("task.autorouting.tiers"); + const changedDeclaration = { schema: 1 as const, providers: ["anthropic", "xai"] }; + let declarationRaceError = ""; + try { + await stableController.applySmartRouting(changedDeclaration, { preview: generated }); + } catch (error) { + declarationRaceError = error instanceof Error ? error.message : String(error); + } + const observed = { + forgedProvenance: forged.provenance, + appliedForged, + persistedForged, + stalePreview, + racedCatalogCalls: catalogCalls, + racedApplied: racedApplied, + racedPersisted: racedPersisted, + declarationRaceError, + }; + const pass = + bytes(persistedForged.provenance) === bytes(generated.provenance) && + bytes(persistedForged.tiers) === bytes(generated.tiers) && + bytes(racedPersisted) === bytes(racedApplied.tiers) && + bytes(racedPersisted) !== bytes(stalePreview.tiers) && + catalogCalls === 2 && + declarationRaceError.length > 0; + record( + "gen2-b1-forged-provenance-race", + "AC6", + rootCommand, + observed, + pass, + "Forged provenance or a declaration/catalog race persisted stale preview bytes.", + ); + expect(pass).toBe(true); + }); + + it("fresh B1 seam: panel preview does not remain stale after controller silently substitutes a raced payload", async () => { + const loadedTheme = await getThemeByName("red-claw"); + if (loadedTheme) setThemeInstance(loadedTheme); + const catalogA = [ + model("anthropic", "claude-haiku-4-5"), + model("anthropic", "claude-sonnet-5"), + model("anthropic", "claude-sonnet-4-6"), + model("anthropic", "claude-opus-5"), + ]; + const catalogB = [model("anthropic", "claude-haiku-4-5")]; + let useCatalogB = false; + const context = testContext(catalogA); + context.ctx.session.modelRegistry = { + getAll: () => (useCatalogB ? catalogB : catalogA), + getAvailable: () => (useCatalogB ? catalogB : catalogA), + }; + const controller = new SelectorController(context.ctx as never); + const setup = { schema: 1 as const, providers: ["anthropic"] }; + const initialPreview = controller.previewSmartRouting(setup); + let applied: ReturnType | undefined; + const panel = new SmartRoutingPanelComponent({ + setup, + enabled: false, + readOnly: false, + stale: false, + preview: initialPreview, + generatePreview: draft => controller.previewSmartRouting(draft), + onSelect: async intent => { + if (intent.kind !== "apply") return; + applied = await controller.applySmartRouting(intent.draft, { preview: intent.preview }); + }, + onCancel: () => undefined, + }); + useCatalogB = true; + await panel.__testApply(); + const panelPreview = panel.getPreviewPayload(); + const observed = { + applied, + panelPreview, + panelMode: panel.mode, + settingsTiers: context.settings.get("task.autorouting.tiers"), + }; + const pass = + panel.mode === "done" && + applied !== undefined && + bytes(panelPreview.tiers) === bytes(applied.tiers) && + bytes(context.settings.get("task.autorouting.tiers")) === bytes(applied.tiers); + record( + "gen2-b1-panel-substitution", + "AC6", + rootCommand, + observed, + pass, + "Controller substituted a regenerated payload but the panel retained stale forged/raced preview bytes after Apply.", + ); + expect(pass).toBe(true); + }); + + it("B2 varied: disabled-present, disabled-unauthenticated, enabled-absent, and ordering remain truthful", async () => { + const catalog = [model("enabled", "present"), model("disabled", "present")]; + const settings = { + "task.autorouting.enabled": true, + "task.autorouting.tiers": { + fast: ["enabled/present", "disabled/present", "disabled/missing", "missing/absent"], + }, + disabledProviders: ["disabled"], + }; + const observedCalls: string[] = []; + const observedRuns: Array<{ skips?: Array<{ selector: string; code: string }>; candidates?: string[] }> = []; + const discover = vi + .spyOn(discoveryModule, "discoverAgents") + .mockResolvedValue({ agents: [taskAgent], projectAgentsDir: null }); + AsyncJobManager.setInstance(new AsyncJobManager({ maxRunningJobs: 4, onJobComplete: async () => {} })); + const runStub = async (options: Parameters[0]) => { + observedRuns.push({ skips: options.autoroutingSkips, candidates: options.autoroutingCandidates }); + return successResult(options); + }; + const getApiKey = async (entry: Model): Promise => { + observedCalls.push(`${entry.provider}/${entry.id}`); + return undefined; + }; + const tool = await TaskTool.create(taskSession(settings, catalog, getApiKey), { runSubprocess: runStub }); + await tool.execute("gen2-skip-order", { + agent: "task", + tasks: [{ id: "one", description: "one", assignment: "run", tier: "fast" }], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + discover.mockRestore(); + const run = observedRuns[0]; + const observed = { run, observedCalls, order: run?.skips?.map(skip => `${skip.selector}:${skip.code}`) }; + const pass = + JSON.stringify(run?.candidates) === JSON.stringify([]) && + JSON.stringify(run?.skips) === + JSON.stringify([ + { selector: "disabled/present", code: "provider_disabled" }, + { selector: "disabled/missing", code: "provider_disabled" }, + { selector: "missing/absent", code: "snapshot_missing" }, + { selector: "enabled/present", code: "credential_unavailable" }, + ]) && + JSON.stringify(observedCalls) === JSON.stringify(["enabled/present"]); + record( + "gen2-b2-skip-ordering", + "AC12", + rootCommand, + observed, + pass, + "Disabled, snapshot-missing, and credential-unavailable candidates were conflated or omitted.", + ); + expect(pass).toBe(true); + }); + + it("B2 one-shot credential lookup faults reach authoritative preflight", async () => { + const catalog = [model("enabled", "present")]; + const settings = { + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: ["enabled/present"] }, + }; + let calls = 0; + let preflightErrors: Map | undefined; + const discover = vi + .spyOn(discoveryModule, "discoverAgents") + .mockResolvedValue({ agents: [taskAgent], projectAgentsDir: null }); + AsyncJobManager.setInstance(new AsyncJobManager({ maxRunningJobs: 4, onJobComplete: async () => {} })); + const tool = await TaskTool.create( + taskSession(settings, catalog, async () => { + calls++; + if (calls === 1) throw new Error("one-shot keychain failure"); + return "key"; + }), + { + runSubprocess: async options => { + preflightErrors = options.autoroutingPreflightErrors; + return successResult(options); + }, + }, + ); + await tool.execute("gen2-one-shot-credential-fault", { + agent: "task", + tasks: [{ id: "one", description: "one", assignment: "run", tier: "fast" }], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + discover.mockRestore(); + expect(calls).toBe(1); + expect(preflightErrors?.get("enabled/present")).toBeInstanceOf(Error); + }); + + it("B3 varied: rejects traversal encodings, absolute/separator ids, unicode ids, and overlong ids before any writer/path effect", async () => { + const invalidIds = [ + "", + ".", + "..", + "../escape", + "../../outside", + "/absolute", + "C:\\\\absolute", + "a/b", + "a\\\\b", + "a/../b", + "a\\\\..\\\\b", + "a\u0000b", + "a\nb", + "e\u0301", + "é", + "%2e%2e", + "a".repeat(129), + ]; + const attempts: Array<{ id: string; error: string; openedPaths: string[]; stagingEntries: string[] }> = []; + for (const id of invalidIds) { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen2-id-")); + const finalPath = path.join(root, "final.jsonl"); + const openedPaths: string[] = []; + const baseStorage = new FileSessionStorage(); + const storage = new Proxy(baseStorage, { + get(target, property, receiver) { + if (property === "openWriter") { + return (filePath: string, options?: Parameters[1]) => { + openedPaths.push(filePath); + return target.openWriter(filePath, options); + }; + } + const value = Reflect.get(target, property, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as unknown as SessionStorage; + let error = ""; + try { + await SessionManager.openStaged(finalPath, storage, id); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + attempts.push({ id, error, openedPaths, stagingEntries: await tree(path.join(root, ".staging")) }); + await rm(root, { recursive: true, force: true }); + } + const validRoot = await mkdtemp(path.join(tmpdir(), "autorouting-gen2-valid-id-")); + const validFinal = path.join(validRoot, "final.jsonl"); + let validAccepted = false; + try { + const valid = await SessionManager.openStaged(validFinal, undefined, "A".repeat(128)); + validAccepted = true; + await valid.discardStaged(); + } finally { + await rm(validRoot, { recursive: true, force: true }); + } + const collisionRoot = await mkdtemp(path.join(tmpdir(), "autorouting-gen2-collision-")); + let collisionError = ""; + try { + await SessionManager.openStaged(path.join(collisionRoot, ".staging", "safe.jsonl"), undefined, "safe"); + } catch (caught) { + collisionError = caught instanceof Error ? caught.message : String(caught); + } + await rm(collisionRoot, { recursive: true, force: true }); + const observed = { invalidIds, attempts, validAccepted, collisionError }; + const pass = + attempts.every( + attempt => + attempt.error.includes("Unsafe artifact attempt id") && + attempt.openedPaths.length === 0 && + attempt.stagingEntries.length === 0, + ) && + validAccepted && + collisionError.includes("Final session path cannot be staged"); + record( + "gen2-b3-attempt-id-variants", + "AC13", + rootCommand, + observed, + pass, + "A traversal, absolute, separator, unicode, overlong, or collision id caused a writer/path effect before rejection.", + ); + expect(pass).toBe(true); + }); + + it("B4 varied: whitespace sanitization and placeholder literals remain bounded and terminal", async () => { + const inputs = [" \t\r\n", "\u0000\u0001", ""]; + let result: SingleResult | undefined; + let error = ""; + try { + result = await runSubprocess({ + cwd: process.cwd(), + agent: taskAgent, + task: "gen2-sanitization", + assignment: "gen2-sanitization", + index: 0, + id: "gen2-sanitization", + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: [], + autoroutingSkips: inputs.map(selector => ({ selector, code: "snapshot_missing" as const })), + routing: { tier: "fast", requestedSelector: "provider/model", substitutions: [] }, + }); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + const selectors = result?.routing?.skips?.map(skip => skip.selector) ?? []; + const observed = { inputs, selectors, terminal: result?.routing?.terminal, error }; + const pass = + error.length === 0 && + result?.routing?.terminal === "all_candidates_skipped" && + selectors.length === 3 && + selectors[0] === " " && + selectors[1] === "" && + selectors[2] === ""; + record( + "gen2-b4-whitespace-placeholder", + "AC12", + rootCommand, + observed, + pass, + "Whitespace-only sanitization threw or escaped the bounded evidence shape; literal placeholder handling failed.", + ); + expect(pass).toBe(true); + }); + + it("B5 varied: exact 256 bounds, 257 rejection, multibyte lengths, and impossible phase/code pairs are enforced", () => { + const makeEvidence = (authResolvedModel: string): TaskRoutingEvidence => ({ + tier: "fast", + requestedSelector: "provider/model", + effectiveModel: "provider/effective", + authResolvedModel, + substitutions: [], + }); + const check = (authResolvedModel: string): string => { + try { + assertRoutingEvidenceInvariant(makeEvidence(authResolvedModel)); + return "accepted"; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + }; + const phaseChecks = [ + (() => { + try { + assertRoutingEvidenceInvariant({ + ...makeEvidence("provider/auth"), + attempts: [{ selector: "provider/model", phase: "probe", code: "post_acceptance_failure" }], + }); + return "accepted"; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + })(), + (() => { + try { + assertRoutingEvidenceInvariant({ + ...makeEvidence("provider/auth"), + attempts: [{ selector: "provider/model", phase: "durable", code: "probe_passed" }], + }); + return "accepted"; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + })(), + ]; + const observed = { + ascii256: check("a".repeat(256)), + ascii257: check("a".repeat(257)), + multibyte256: check("é".repeat(256)), + multibyte257: check("é".repeat(257)), + phaseChecks, + }; + const pass = + observed.ascii256 === "accepted" && + observed.ascii257 !== "accepted" && + observed.multibyte256 === "accepted" && + observed.multibyte257 !== "accepted" && + phaseChecks.every(message => message !== "accepted"); + record( + "gen2-b5-bounds-phase-pairs", + "AC12", + rootCommand, + observed, + pass, + "A 257-length/multibyte auth-resolved model was accepted, a 256-length model rejected, or impossible phase/code pairing escaped.", + ); + expect(pass).toBe(true); + }); + + it("manual-fallback skip evidence is bounded and aggregated before invariant validation", () => { + const skips = Array.from({ length: 20 }, (_, index) => ({ + selector: index === 0 ? "provider/\u0000model" : `provider/${"x".repeat(300)}-${index}`, + code: "snapshot_missing" as const, + })); + const projection = buildBoundedRoutingSkips(skips); + const evidence: TaskRoutingEvidence = { + tier: "fast", + requestedSelector: "manual-model-chain", + effectiveModel: "manual-model-chain", + substitutions: [], + ...projection, + }; + assertRoutingEvidenceInvariant(evidence); + record( + "gen3-direct-skip-projection", + "AC12", + rootCommand, + { projection }, + true, + "Direct bounded-skip projection did not preserve selector bounds or omitted-code aggregates.", + ); + expect(projection.skips).toHaveLength(16); + expect(projection.skips?.[0]?.selector).toBe("provider/model"); + expect(projection.omittedSkipCount).toBe(4); + expect(projection.omittedByCode).toEqual({ snapshot_missing: 4 }); + }); + + it("fresh post-fence seams: structural remap is simultaneous, deferred rollback removes only owned publication, and finalize is stable", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen2-publisher-")); + const finalPath = path.join(root, "candidate.jsonl"); + const parentDir = path.join(root, "candidate"); + const parentArtifacts = new ArtifactManager(parentDir); + await parentArtifacts.save("sibling", "tool"); + const manager = await SessionManager.openStaged(finalPath, undefined, "publisher-gen2"); + const stagedArtifacts = manager.getArtifactManager(); + if (!stagedArtifacts) throw new Error("staged artifact manager unavailable"); + const first = await stagedArtifacts.save("first", "tool"); + const second = await stagedArtifacts.save("second", "tool"); + manager.appendCustomEntry("refs", { + id: first, + parentId: first, + timestamp: first, + refs: [`artifact://${first}`, `artifact://${second}`, `agent://${first}`, `agent://${second}`], + nested: { first: first, second: second, adjacent: `${first}${second}` }, + decimal: "0.5", + }); + await manager.commitStaged({ deferArtifactFinalize: true }); + const publishedText = await Bun.file(finalPath).text(); + const publishedTree = await tree(parentDir); + await manager.rollbackCommittedStaged(); + const rolledBack = { + final: await fileBytes(finalPath), + parentTree: await tree(parentDir), + stagingTree: await tree(path.join(root, ".staging")), + }; + const hookRoot = await mkdtemp(path.join(tmpdir(), "autorouting-gen2-hook-")); + const hookParent = new ArtifactManager(path.join(hookRoot, "parent")); + await hookParent.save("sibling", "tool"); + const hookParentBefore = await tree(hookParent.dir); + const hookStaging = hookParent.createAttemptStaging("before-hook"); + await hookStaging.save("candidate", "tool"); + const hookBefore = await tree(hookParent.dir); + let hookError = ""; + try { + await hookParent.commitAttemptStaging(hookStaging, "before-hook", { + beforePublish: () => { + throw new Error("before-publish-injected"); + }, + }); + } catch (error) { + hookError = error instanceof Error ? error.message : String(error); + } + await hookStaging.discardAttemptStaging(); + const hookAfter = await tree(hookParent.dir); + const finalizeRoot = await mkdtemp(path.join(tmpdir(), "autorouting-gen2-finalize-")); + const finalizePath = path.join(finalizeRoot, "candidate.jsonl"); + const finalizeManager = await SessionManager.openStaged(finalizePath, undefined, "finalize-gen2"); + const finalizeArtifacts = finalizeManager.getArtifactManager(); + if (!finalizeArtifacts) throw new Error("finalize artifact manager unavailable"); + await finalizeArtifacts.save("finalize", "tool"); + await finalizeManager.commitStaged({ deferArtifactFinalize: true }); + const beforeFinalize = { + final: await fileBytes(finalizePath), + tree: await tree(path.join(finalizeRoot, "candidate")), + }; + finalizeManager.finalizeStagedCommit(); + const afterFinalize = { + final: await fileBytes(finalizePath), + tree: await tree(path.join(finalizeRoot, "candidate")), + }; + await rm(root, { recursive: true, force: true }); + await rm(hookRoot, { recursive: true, force: true }); + await rm(finalizeRoot, { recursive: true, force: true }); + const observed = { + first, + second, + publishedText, + publishedTree, + rolledBack, + hookParentBefore, + hookBefore, + hookError, + hookAfter, + beforeFinalize, + afterFinalize, + }; + const pass = + publishedText.includes("artifact://1") && + publishedText.includes("artifact://2") && + publishedText.includes("agent://1") && + publishedText.includes("agent://2") && + publishedText.includes('"decimal":"0.5"') && + publishedText.includes(`"id":"${first}"`) && + rolledBack.final === null && + JSON.stringify(rolledBack.parentTree) === JSON.stringify([".artifact-id-0", "0.tool.log"]) && + rolledBack.stagingTree.length === 0 && + hookError === "before-publish-injected" && + JSON.stringify(hookParentBefore) === JSON.stringify(hookAfter) && + bytes(beforeFinalize.final) === bytes(afterFinalize.final) && + JSON.stringify(beforeFinalize.tree) === JSON.stringify(afterFinalize.tree); + record( + "gen2-post-fence-remap-rollback", + "AC13", + rootCommand, + observed, + pass, + "Post-fence remap double-substituted chained IDs, deferred rollback leaked owned files, or finalize changed publication bytes.", + ); + expect(pass).toBe(true); + }); + it("deferred publication leaves breadcrumb untouched until finalization and aggregates cleanup failures", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen3-publisher-")); + const finalPath = path.join(root, "candidate.jsonl"); + const beforeBreadcrumb = await terminalBreadcrumbBytes(); + const manager = await SessionManager.openStaged(finalPath, undefined, "gen3-breadcrumb"); + const artifacts = manager.getArtifactManager(); + if (!artifacts) throw new Error("staged artifact manager unavailable"); + await artifacts.save("candidate", "tool"); + await manager.commitStaged({ deferArtifactFinalize: true }); + const afterDeferredCommit = await terminalBreadcrumbBytes(); + await manager.rollbackCommittedStaged(); + const afterRollback = await terminalBreadcrumbBytes(); + + const collisionRoot = await mkdtemp(path.join(tmpdir(), "autorouting-gen3-cleanup-")); + const collisionPath = path.join(collisionRoot, "candidate.jsonl"); + const collisionManager = await SessionManager.openStaged(collisionPath, undefined, "gen3-cleanup"); + const collisionArtifacts = collisionManager.getArtifactManager(); + if (!collisionArtifacts) throw new Error("collision artifact manager unavailable"); + await collisionArtifacts.save("candidate", "tool"); + await writeFile(collisionPath, "pre-existing-final"); + const cleanupFailure = new Error("discard-cleanup-failed"); + vi.spyOn(collisionManager, "discardStaged").mockRejectedValue(cleanupFailure); + let commitFailure: unknown; + try { + await collisionManager.commitStaged(); + } catch (error) { + commitFailure = error; + } + const discardRoot = await mkdtemp(path.join(tmpdir(), "autorouting-gen3-discard-")); + const discardManager = await SessionManager.openStaged( + path.join(discardRoot, "candidate.jsonl"), + undefined, + "gen3-discard", + ); + const discardArtifacts = discardManager.getArtifactManager(); + if (!discardArtifacts) throw new Error("discard artifact manager unavailable"); + await discardArtifacts.save("candidate", "tool"); + const discardCleanupFailure = new Error("preflight-discard-cleanup-failed"); + vi.spyOn(discardArtifacts, "discardAttemptStaging").mockRejectedValue(discardCleanupFailure); + let discardFailure: unknown; + try { + await discardManager.discardStaged(); + } catch (error) { + discardFailure = error; + } + const discardCauses = discardFailure instanceof AggregateError ? discardFailure.errors : []; + const causes = commitFailure instanceof AggregateError ? commitFailure.errors : []; + const pass = + afterDeferredCommit === beforeBreadcrumb && + afterRollback === beforeBreadcrumb && + commitFailure instanceof AggregateError && + causes.some(cause => cause === cleanupFailure) && + causes.length >= 2 && + discardFailure instanceof AggregateError && + discardCauses.some(cause => cause === discardCleanupFailure); + record( + "gen3-breadcrumb-cleanup-evidence", + "AC13", + rootCommand, + { + beforeBreadcrumb, + afterDeferredCommit, + afterRollback, + commitFailure, + causes: causes.map(String), + discardFailure, + discardCauses: discardCauses.map(String), + }, + pass, + "Deferred publication changed the continue breadcrumb before post-commit success, or cleanup failure masked the original commit failure.", + ); + expect(pass).toBe(true); + await rm(root, { recursive: true, force: true }); + await rm(collisionRoot, { recursive: true, force: true }); + await rm(discardRoot, { recursive: true, force: true }); + }); +}); + +describe("autorouting boundary red-team generation 3 delta re-attacks", () => { + it("B6 varied callback failures, races, and rapid Apply calls stay canonical", async () => { + const catalogA = [ + model("anthropic", "claude-haiku-4-5"), + model("anthropic", "claude-sonnet-5"), + model("anthropic", "claude-sonnet-4-6"), + model("anthropic", "claude-opus-5"), + ]; + const catalogB = [model("anthropic", "claude-haiku-4-5")]; + const setup = { schema: 1 as const, providers: ["anthropic"] }; + const context = (useB: () => boolean) => { + const result = testContext(catalogA); + result.ctx.session.modelRegistry = { + getAll: () => (useB() ? catalogB : catalogA), + getAvailable: () => (useB() ? catalogB : catalogA), + }; + return result; + }; + let voidUseB = false; + const voidContext = context(() => voidUseB); + const voidController = new SelectorController(voidContext.ctx as never); + const voidPanel = new SmartRoutingPanelComponent({ + setup, + enabled: false, + readOnly: false, + stale: false, + preview: voidController.previewSmartRouting(setup), + generatePreview: draft => voidController.previewSmartRouting(draft), + onSelect: async intent => { + if (intent.kind === "apply") + await voidController.applySmartRouting(intent.draft, { preview: intent.preview }); + }, + onCancel: () => undefined, + }); + voidUseB = true; + await voidPanel.__testApply(); + const voidApplied = { + preview: voidPanel.getPreviewPayload(), + tiers: voidContext.settings.get("task.autorouting.tiers"), + mode: voidPanel.mode, + }; + let productionUseB = false; + const productionContext = context(() => productionUseB); + const productionController = new SelectorController(productionContext.ctx as never); + const productionPanel = new SmartRoutingPanelComponent({ + setup, + enabled: false, + readOnly: false, + stale: false, + preview: productionController.previewSmartRouting(setup), + generatePreview: draft => productionController.previewSmartRouting(draft), + onSelect: async intent => + intent.kind === "apply" + ? productionController.applySmartRouting(intent.draft, { preview: intent.preview }) + : undefined, + onCancel: () => undefined, + }); + productionUseB = true; + await productionPanel.__testApply(); + const productionApplied = { + preview: productionPanel.getPreviewPayload(), + tiers: productionContext.settings.get("task.autorouting.tiers"), + mode: productionPanel.mode, + }; + const failureContext = testContext(catalogA); + const failureInitial = new SelectorController(failureContext.ctx as never).previewSmartRouting(setup); + const failure = new Error("apply-failure-injected"); + const failurePanel = new SmartRoutingPanelComponent({ + setup, + enabled: false, + readOnly: false, + stale: false, + preview: failureInitial, + generatePreview: () => failureInitial, + onSelect: async () => { + throw failure; + }, + onCancel: () => undefined, + }); + await failurePanel.__testApply(); + let releaseRapid!: () => void; + let rapidCalls = 0; + const rapidContext = testContext(catalogA); + const rapidInitial = new SelectorController(rapidContext.ctx as never).previewSmartRouting(setup); + const rapidPanel = new SmartRoutingPanelComponent({ + setup, + enabled: false, + readOnly: false, + stale: false, + preview: rapidInitial, + generatePreview: () => rapidInitial, + onSelect: async () => { + rapidCalls++; + await new Promise(resolve => { + releaseRapid = resolve; + }); + return rapidInitial; + }, + onCancel: () => undefined, + }); + const firstApply = rapidPanel.__testApply(); + await Promise.resolve(); + const modeDuringRapid = rapidPanel.mode; + const secondApply = rapidPanel.__testApply(); + releaseRapid(); + await Promise.all([firstApply, secondApply]); + const observed = { + voidApplied, + productionApplied, + failure: { + mode: failurePanel.mode, + preview: failurePanel.getPreviewPayload(), + tiers: failureContext.settings.get("task.autorouting.tiers"), + }, + rapid: { rapidCalls, modeDuringRapid, mode: rapidPanel.mode, preview: rapidPanel.getPreviewPayload() }, + }; + const pass = + voidApplied.mode === "done" && + bytes(voidApplied.preview.tiers) === bytes(voidApplied.tiers) && + voidApplied.preview.tiers.balanced === undefined && + productionApplied.mode === "done" && + bytes(productionApplied.preview.tiers) === bytes(productionApplied.tiers) && + productionApplied.preview.tiers.balanced === undefined && + observed.failure.mode === "error" && + bytes(observed.failure.preview) === bytes(failureInitial) && + bytes(observed.failure.tiers) === bytes({}) && + observed.rapid.rapidCalls === 1 && + observed.rapid.modeDuringRapid === "committing" && + observed.rapid.mode === "done" && + bytes(observed.rapid.preview) === bytes(rapidInitial); + record( + "gen3-b6-callback-and-rapid", + "AC6", + rootCommand, + observed, + pass, + "Void/production callbacks diverged, Apply failure entered done or mutated preview, or rapid Apply calls interleaved.", + ); + expect(pass).toBe(true); + }); + + it("B7 varied cycle/substrings/URI nesting preserve one-pass field-aware remap", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen3-remap-")); + const manager = await SessionManager.openStaged( + path.join(root, "candidate.jsonl"), + undefined, + "gen3-remap-cycle", + ); + const original = { + artifactRef: "artifact://0/path/1?next=11#frag", + artifactRefs: ["artifact://0", "artifact://1", "artifact://11", "agent://111/path"], + artifactId: "0", + artifactIds: [0, 1, 11, 111], + nested: { artifactRef: "artifact://1/nested", artifactRefs: [["agent://0"], ["artifact://11?x=1"]] }, + selectorTail: "artifact://0:1-100", + selectorTailCompound: "artifact://11:raw:1-100", + selectorTailAgent: "agent://1:raw", + ordinaryDecimal: "0.5", + ordinaryText: "prefix artifact://0 suffix", + ordinaryAgentText: "agent://11 is only prose", + }; + manager.appendCustomEntry("gen3-remap", original); + const remap = new Map([ + ["0", "1"], + ["1", "0"], + ["11", "1"], + ["111", "11"], + ]); + await manager.remapStagedArtifactReferences(remap); + const stagedFile = manager.getSessionFile(); + const stagedText = stagedFile ? await Bun.file(stagedFile).text() : ""; + const custom = stagedText + .split("\n") + .filter(Boolean) + .map(line => JSON.parse(line) as { customType?: string; data?: typeof original }) + .find(entry => entry.customType === "gen3-remap"); + await manager.discardStaged(); + await rm(root, { recursive: true, force: true }); + const remapped = custom?.data; + const observed = { remap: [...remap.entries()], remapped, stagedFile, stagedText }; + const pass = + remapped?.artifactRef === "artifact://1/path/1?next=11#frag" && + JSON.stringify(remapped?.artifactRefs) === + JSON.stringify(["artifact://1", "artifact://0", "artifact://1", "agent://11/path"]) && + remapped?.artifactId === "1" && + JSON.stringify(remapped?.artifactIds) === JSON.stringify([1, 0, 1, 11]) && + remapped?.nested?.artifactRef === "artifact://0/nested" && + JSON.stringify(remapped?.nested?.artifactRefs) === JSON.stringify([["agent://1"], ["artifact://1?x=1"]]) && + remapped?.ordinaryDecimal === "0.5" && + remapped?.selectorTail === "artifact://1:1-100" && + remapped?.selectorTailCompound === "artifact://1:raw:1-100" && + remapped?.selectorTailAgent === "agent://0:raw" && + remapped?.ordinaryText === "prefix artifact://0 suffix" && + remapped?.ordinaryAgentText === "agent://11 is only prose"; + record( + "gen3-b7-remap-cycle", + "AC13", + rootCommand, + observed, + pass, + "Cycle/substrings cascaded, nested/array URI segments were not one-pass, or non-reference lookalike text was rewritten.", + ); + expect(pass).toBe(true); + }); + + it("shared bounded-skip projection matches routed preflight and manual fallback", async () => { + const catalog = [model("present", "model")]; + const longMissing = (index: number) => `missing/${"x".repeat(200)}-${index}`; + const missingSelectors = Array.from({ length: 20 }, (_, index) => longMissing(index)); + const discover = vi + .spyOn(discoveryModule, "discoverAgents") + .mockResolvedValue({ agents: [taskAgent], projectAgentsDir: null }); + AsyncJobManager.setInstance(new AsyncJobManager({ maxRunningJobs: 4, onJobComplete: async () => {} })); + const runOnce = async (selectors: string[]) => { + const settingsOverrides = { "task.autorouting.enabled": true, "task.autorouting.tiers": { fast: selectors } }; + const captured: Array<{ + routing?: TaskRoutingEvidence; + autoroutingCandidates?: string[]; + autoroutingPreflight?: boolean; + }> = []; + const tool = await TaskTool.create( + taskSession(settingsOverrides, catalog, async () => "key"), + { + runSubprocess: async options => { + captured.push({ + routing: options.routing, + autoroutingCandidates: options.autoroutingCandidates, + autoroutingPreflight: options.autoroutingPreflight, + }); + return successResult(options); + }, + }, + ); + await tool.execute("gen3-skip-projection", { + agent: "task", + tasks: [{ id: "one", description: "one", assignment: "run", tier: "fast" }], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + return captured[0]; + }; + const preflight = await runOnce(["present/model", ...missingSelectors]); + const manual = await runOnce(missingSelectors); + discover.mockRestore(); + const observed = { preflight, manual }; + const preflightSkips = preflight?.routing?.skips; + const manualSkips = manual?.routing?.skips; + const pass = + preflight?.autoroutingPreflight === true && + manual?.autoroutingPreflight === false && + JSON.stringify(preflightSkips) === JSON.stringify(manualSkips) && + preflightSkips?.length === 16 && + manualSkips?.length === 16 && + preflight?.routing?.omittedSkipCount === 4 && + manual?.routing?.omittedSkipCount === 4 && + preflightSkips.every(skip => skip.selector.length <= 256) && + manualSkips.every(skip => skip.selector.length <= 256) && + preflight?.routing?.omittedByCode?.snapshot_missing === 4 && + manual?.routing?.omittedByCode?.snapshot_missing === 4; + record( + "gen3-shared-skip-projection", + "AC12", + rootCommand, + observed, + pass, + "Manual fallback and routed preflight diverged in retained/omitted skip evidence or selector bounds.", + ); + expect(pass).toBe(true); + }); + + it("deferred breadcrumb ordering and AggregateError retain original failures", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen3-interleave-")); + const finalPath = path.join(root, "candidate.jsonl"); + const before = await terminalBreadcrumbBytes(); + const manager = await SessionManager.openStaged(finalPath, undefined, "gen3-interleave"); + const artifacts = manager.getArtifactManager(); + if (!artifacts) throw new Error("artifact manager unavailable"); + await artifacts.save("candidate", "tool"); + await manager.commitStaged({ deferArtifactFinalize: true }); + const deferred = await terminalBreadcrumbBytes(); + await manager.rollbackCommittedStaged(); + const rolledBack = await terminalBreadcrumbBytes(); + manager.finalizeStagedCommit(); + const afterNoopFinalize = await terminalBreadcrumbBytes(); + const aggregateRoot = await mkdtemp(path.join(tmpdir(), "autorouting-gen3-aggregate-")); + const aggregatePath = path.join(aggregateRoot, "candidate.jsonl"); + const aggregateManager = await SessionManager.openStaged(aggregatePath, undefined, "gen3-aggregate"); + const aggregateArtifacts = aggregateManager.getArtifactManager(); + if (!aggregateArtifacts) throw new Error("aggregate artifact manager unavailable"); + await aggregateArtifacts.save("candidate", "tool"); + await writeFile(aggregatePath, "existing-final"); + const cleanupFailure = new Error("aggregate-discard-cleanup-failed"); + vi.spyOn(aggregateManager, "discardStaged").mockRejectedValue(cleanupFailure); + let aggregateFailure: unknown; + try { + await aggregateManager.commitStaged(); + } catch (error) { + aggregateFailure = error; + } + const aggregateErrors = aggregateFailure instanceof AggregateError ? aggregateFailure.errors : []; + const aggregateStrings = aggregateErrors.map(String); + const observed = { before, deferred, rolledBack, afterNoopFinalize, aggregateFailure, aggregateStrings }; + const pass = + deferred === before && + rolledBack === before && + afterNoopFinalize === before && + aggregateFailure instanceof AggregateError && + aggregateErrors.some(error => error !== cleanupFailure) && + aggregateErrors.some(error => error === cleanupFailure) && + aggregateErrors.length >= 2; + record( + "gen3-breadcrumb-aggregate-interleaving", + "AC13", + rootCommand, + observed, + pass, + "Deferred commit/rollback/finalize changed breadcrumb ordering, or AggregateError masked the original publication failure.", + ); + expect(pass).toBe(true); + await rm(root, { recursive: true, force: true }); + await rm(aggregateRoot, { recursive: true, force: true }); + }); +}); + +describe("autorouting boundary red-team generation 4 delta re-attacks", () => { + it("B8 selector-tail remap follows the internal URL grammar across malformed and compound tails", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen4-selector-tail-")); + const manager = await SessionManager.openStaged( + path.join(root, "candidate.jsonl"), + undefined, + "gen4-selector-tail", + ); + const tailOriginal = { + artifactSelectors: [ + "artifact://3:raw:1-100", + "artifact://3:1-100:raw", + "artifact://3:", + "artifact://3:bogus", + "artifact://3:-100", + "artifact://1:1-1", + "artifact://11:1-1", + ], + agentSelectors: ["agent://3:raw:1-100", "agent://3:1-100:raw", "agent://3:raw"], + malformedAgent: ["agent://3:", "agent://3:bogus", "agent://3:-100", "agent://3:raw:bogus"], + otherSchemes: ["local://3:raw", "memory://3:1-100", "rule://3:raw"], + nested: { arrays: [["artifact://1:1-1", "artifact://11:1-1"], [{ artifactRef: "agent://1:raw" }]] }, + prose: { + validArtifact: "Read artifact://3:1-100 now", + invalidAgent: "prefix agent://3:raw suffix", + otherScheme: "prefix local://3:raw suffix", + }, + }; + const cycleOriginal = { + nested: { + array: ["artifact://1:1-1", "artifact://11:1-1", ["agent://1:raw", "artifact://30:raw"]], + deep: { artifactRef: "artifact://3/path", prose: "prefix artifact://30:raw suffix" }, + }, + id: "artifact://1:1-1", + structural: { id: "artifact://11:1-1", parentId: "agent://3:raw", timestamp: "artifact://3:" }, + }; + manager.appendCustomEntry("gen4-selector-tail", tailOriginal); + manager.appendCustomEntry("gen4-cycle", cycleOriginal); + const remap = new Map([ + ["1", "11"], + ["11", "1"], + ["3", "30"], + ["30", "3"], + ]); + const parserInputs = [ + ...tailOriginal.artifactSelectors, + ...tailOriginal.agentSelectors, + ...tailOriginal.malformedAgent, + ...tailOriginal.otherSchemes, + ]; + const parserObserved = Object.fromEntries(parserInputs.map(value => [value, splitInternalUrlSel(value)])); + await manager.remapStagedArtifactReferences(remap); + const stagedFile = manager.getSessionFile(); + const stagedText = stagedFile ? await Bun.file(stagedFile).text() : ""; + const customEntries = stagedText + .split("\n") + .filter(Boolean) + .map(line => JSON.parse(line) as { customType?: string; data?: unknown }); + const remappedTail = customEntries.find(entry => entry.customType === "gen4-selector-tail")?.data as + | typeof tailOriginal + | undefined; + const remappedCycle = customEntries.find(entry => entry.customType === "gen4-cycle")?.data as + | typeof cycleOriginal + | undefined; + await manager.discardStaged(); + await rm(root, { recursive: true, force: true }); + const observed = { + remap: [...remap.entries()], + parserObserved, + remappedTail, + remappedCycle, + stagedFile, + }; + const tailPass = + JSON.stringify(remappedTail?.artifactSelectors) === + JSON.stringify([ + "artifact://30:raw:1-100", + "artifact://30:1-100:raw", + "artifact://30:", + "artifact://30:bogus", + "artifact://30:-100", + "artifact://11:1-1", + "artifact://1:1-1", + ]) && + JSON.stringify(remappedTail?.agentSelectors) === + JSON.stringify(["agent://30:raw:1-100", "agent://30:1-100:raw", "agent://30:raw"]) && + JSON.stringify(remappedTail?.malformedAgent) === + JSON.stringify(["agent://3:", "agent://3:bogus", "agent://3:-100", "agent://3:raw:bogus"]) && + JSON.stringify(remappedTail?.otherSchemes) === + JSON.stringify(["local://3:raw", "memory://3:1-100", "rule://3:raw"]) && + JSON.stringify(remappedTail?.nested) === + JSON.stringify({ + arrays: [["artifact://11:1-1", "artifact://1:1-1"], [{ artifactRef: "agent://11:raw" }]], + }) && + // Final rule: re-keying happens only when the ENTIRE value is a single URI token, so a URI + // embedded in a prose sentence is never rewritten — for either scheme. This supersedes an + // earlier reconciliation that remapped artifact-in-prose but not agent-in-prose; the + // stricter whole-token rule is consistent, is what the implementation now enforces, and + // removes the whole class of over-rewrite bugs found in earlier generations. + remappedTail?.prose.validArtifact === "Read artifact://3:1-100 now" && + remappedTail?.prose.invalidAgent === "prefix agent://3:raw suffix" && + remappedTail?.prose.otherScheme === "prefix local://3:raw suffix" && + parserObserved["artifact://3:"]?.path === "artifact://3" && + parserObserved["artifact://3:"]?.sel === "" && + parserObserved["artifact://3:bogus"]?.sel === "bogus" && + parserObserved["artifact://3:-100"]?.sel === "-100" && + parserObserved["agent://3:raw:1-100"]?.sel === "raw:1-100" && + parserObserved["agent://3:bogus"]?.sel === undefined; + const cyclePass = + JSON.stringify(remappedCycle?.nested) === + JSON.stringify({ + array: ["artifact://11:1-1", "artifact://1:1-1", ["agent://11:raw", "artifact://3:raw"]], + // prose is a multi-token string, so the whole-token rule leaves it verbatim. + deep: { artifactRef: "artifact://30/path", prose: "prefix artifact://30:raw suffix" }, + }) && + remappedCycle?.id === "artifact://1:1-1" && + JSON.stringify(remappedCycle?.structural) === + JSON.stringify({ id: "artifact://11:1-1", parentId: "agent://3:raw", timestamp: "artifact://3:" }); + record( + "gen4-b8-selector-tail-parser", + "AC13", + rootCommand, + observed, + tailPass, + "Selector-tail remapping diverged from splitInternalUrlSel: malformed agent tails, non-artifact schemes, nested arrays, prose, or substring IDs were rewritten incorrectly.", + ); + record( + "gen4-b9-remap-cycle-nested", + "AC13", + rootCommand, + observed, + cyclePass, + "A cyclic ID map cascaded, structural keys were rewritten, or nested selector-tail references were not remapped in one pass.", + ); + expect(tailPass && cyclePass).toBe(true); + }); + + it("C1 pre-fence discard failure fails closed while retaining both primary and cleanup evidence", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen4-cleanup-failure-")); + const finalPath = path.join(root, "candidate.jsonl"); + const primary = Object.assign(new Error("transient bootstrap evidence"), { transient: true }); + const cleanup = new Error("discard-cleanup-failed evidence"); + const modelEntry = model("test", "model"); + const createSpy = vi.spyOn(sdkModule, "createAgentSession").mockRejectedValue(primary); + const discardSpy = vi.spyOn(SessionManager.prototype, "discardStaged").mockRejectedValueOnce(cleanup); + let result: Awaited> | undefined; + try { + result = await runSubprocessOnce({ + cwd: root, + agent: taskAgent, + task: "gen4 cleanup failure", + assignment: "gen4 cleanup failure", + index: 0, + id: "gen4-cleanup-failure", + modelOverride: ["test/model"], + settings: Settings.isolated(), + modelRegistry: { + authStorage: {}, + getAvailable: () => [modelEntry], + getApiKey: async () => "key", + } as never, + preflightDurable: true, + autoroutingAttemptId: "gen4-cleanup-failure", + sessionFile: finalPath, + }); + } finally { + discardSpy.mockRestore(); + createSpy.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + const observed = { + preflightFenceCrossed: result?.preflightFenceCrossed, + preflightFailure: result?.preflightFailure, + error: result?.error, + primary: primary.message, + cleanup: cleanup.message, + }; + const pass = + result?.preflightFenceCrossed === false && + JSON.stringify(result?.preflightFailure) === + JSON.stringify({ kind: "local", op: "preflight_validation", transient: false }) && + (result?.error ?? "").includes(primary.message) && + (result?.error ?? "").includes(cleanup.message); + record( + "gen4-c1-cleanup-fail-closed", + "AC13", + rootCommand, + observed, + pass, + "Pre-fence cleanup failure did not downgrade to terminal or caused the original setup failure evidence to disappear.", + ); + expect(pass).toBe(true); + }); + + it("C2 a normal transient durable failure with successful discard advances to the next unique candidate", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen4-successful-discard-")); + const finalPath = path.join(root, "candidate.jsonl"); + const models = [ + model("test", "first", true, { + headers: {}, + compat: {}, + thinking: { mode: "effort", minLevel: "minimal", maxLevel: "high" }, + }), + model("test", "second", true, { + headers: {}, + compat: {}, + thinking: { mode: "effort", minLevel: "minimal", maxLevel: "high" }, + }), + ]; + const authStorage = await AuthStorage.create(":memory:"); + const modelRegistry = new ModelRegistry(authStorage); + vi.spyOn(modelRegistry, "getAll").mockReturnValue(models); + vi.spyOn(modelRegistry, "getAvailable").mockReturnValue(models); + vi.spyOn(modelRegistry, "getApiKey").mockImplementation(async () => "key"); + const transientFailure = Object.assign(new Error("normal transient durable failure"), { transient: true }); + const originalCreate = sdkModule.createAgentSession; + let createCalls = 0; + const createErrors: string[] = []; + const createSpy = vi.spyOn(sdkModule, "createAgentSession").mockImplementation(async options => { + createCalls++; + if (createCalls === 2) throw transientFailure; + try { + const result = await originalCreate(options); + vi.spyOn(result.session, "prompt").mockImplementation(async (_message, promptOptions) => { + if (promptOptions?.onPreflightAcceptCommit) await promptOptions.onPreflightAcceptCommit(); + else promptOptions?.onPreflightAccepted?.(); + if (createCalls === 4) + result.session.agent.emitExternalEvent({ + type: "tool_execution_end", + toolCallId: "gen4-yield", + toolName: "yield", + result: { content: [], details: { status: "success", data: {} } }, + isError: false, + } as never); + }); + vi.spyOn(result.session, "waitForIdle").mockResolvedValue(undefined); + return result; + } catch (error) { + createErrors.push(error instanceof Error ? error.message : String(error)); + throw error; + } + }); + let result: Awaited> | undefined; + let stagingTree: string[] = []; + let finalExists = false; + try { + result = await runSubprocess({ + cwd: root, + agent: taskAgent, + task: "gen4 successful discard", + assignment: "gen4 successful discard", + index: 0, + id: "gen4-successful-discard", + modelOverride: ["test/first", "test/second"], + settings: Settings.isolated(), + modelRegistry, + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: ["test/first", "test/second"], + autoroutingSkips: [], + routing: { + tier: "fast", + requestedSelector: "test/first", + effectiveModel: "test/first", + substitutions: [], + }, + sessionFile: finalPath, + }); + stagingTree = await tree(path.join(root, ".staging")); + finalExists = await fileBytes(finalPath).then(value => value !== null); + } finally { + createSpy.mockRestore(); + authStorage.close(); + await rm(root, { recursive: true, force: true }); + } + const expectedAttempts = [ + { selector: "test/first", phase: "probe", code: "probe_passed" }, + { selector: "test/first", phase: "durable", code: "spawn_transient_retry" }, + { selector: "test/second", phase: "probe", code: "probe_passed" }, + { selector: "test/second", phase: "durable", code: "accepted" }, + ]; + const observed = { + createCalls, + createErrors, + attempts: result?.routing?.attempts, + stagingTree, + finalExists, + exitCode: result?.exitCode, + firstError: result?.error, + setupFailure: result?.setupFailure, + preflightFenceCrossed: result?.preflightFenceCrossed, + }; + const pass = + createCalls === 4 && + JSON.stringify(result?.routing?.attempts) === JSON.stringify(expectedAttempts) && + stagingTree.length === 0 && + finalExists && + result?.exitCode === 0 && + result?.preflightFenceCrossed === true; + record( + "gen4-c2-successful-discard-advances", + "AC13", + rootCommand, + observed, + pass, + "A successful pre-fence discard failed to advance the normal transient candidate, leaked staging residue, or skipped the accepted ledger entry.", + ); + expect(pass).toBe(true); + }, 30_000); + + it("C3 a post-fence failure stays terminal and never enters the pre-fence discard downgrade", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen4-post-fence-")); + const finalPath = path.join(root, "candidate.jsonl"); + const models = [ + model("test", "first", true, { + headers: {}, + compat: {}, + thinking: { mode: "effort", minLevel: "minimal", maxLevel: "high" }, + }), + model("test", "second", true, { + headers: {}, + compat: {}, + thinking: { mode: "effort", minLevel: "minimal", maxLevel: "high" }, + }), + ]; + const authStorage = await AuthStorage.create(":memory:"); + const modelRegistry = new ModelRegistry(authStorage); + vi.spyOn(modelRegistry, "getAll").mockReturnValue(models); + vi.spyOn(modelRegistry, "getAvailable").mockReturnValue(models); + vi.spyOn(modelRegistry, "getApiKey").mockImplementation(async () => "key"); + const postFenceFailure = new Error("post-fence failure evidence"); + const originalCreate = sdkModule.createAgentSession; + let createCalls = 0; + const createErrors: string[] = []; + const createSpy = vi.spyOn(sdkModule, "createAgentSession").mockImplementation(async options => { + createCalls++; + if (createCalls > 2) throw new Error("unexpected candidate advance"); + try { + const result = await originalCreate(options); + vi.spyOn(result.session, "prompt").mockImplementation(async (_message, promptOptions) => { + if (promptOptions?.onPreflightAcceptCommit) await promptOptions.onPreflightAcceptCommit(); + else promptOptions?.onPreflightAccepted?.(); + if (createCalls === 2) throw postFenceFailure; + }); + vi.spyOn(result.session, "waitForIdle").mockResolvedValue(undefined); + return result; + } catch (error) { + createErrors.push(error instanceof Error ? error.message : String(error)); + throw error; + } + }); + let discardCalls = 0; + const originalDiscard = SessionManager.prototype.discardStaged; + const discardSpy = vi.spyOn(SessionManager.prototype, "discardStaged").mockImplementation(async function ( + this: SessionManager, + ) { + discardCalls++; + return originalDiscard.call(this); + }); + let result: Awaited> | undefined; + let stagingTree: string[] = []; + let finalExists = false; + try { + result = await runSubprocess({ + cwd: root, + agent: taskAgent, + task: "gen4 post-fence failure", + assignment: "gen4 post-fence failure", + index: 0, + id: "gen4-post-fence", + modelOverride: ["test/first", "test/second"], + settings: Settings.isolated(), + modelRegistry, + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: ["test/first", "test/second"], + autoroutingSkips: [], + routing: { + tier: "fast", + requestedSelector: "test/first", + effectiveModel: "test/first", + substitutions: [], + }, + sessionFile: finalPath, + }); + stagingTree = await tree(path.join(root, ".staging")); + finalExists = await fileBytes(finalPath).then(value => value !== null); + } finally { + discardSpy.mockRestore(); + createSpy.mockRestore(); + authStorage.close(); + await rm(root, { recursive: true, force: true }); + } + const observed = { + createCalls, + createErrors, + discardCalls, + attempts: result?.routing?.attempts, + preflightFailure: result?.preflightFailure, + preflightFenceCrossed: result?.preflightFenceCrossed, + stagingTree, + finalExists, + error: result?.error, + setupFailure: result?.setupFailure, + }; + const pass = + createCalls === 2 && + discardCalls === 0 && + JSON.stringify(result?.routing?.attempts) === + JSON.stringify([ + { selector: "test/first", phase: "probe", code: "probe_passed" }, + { selector: "test/first", phase: "durable", code: "post_acceptance_failure" }, + ]) && + result?.preflightFenceCrossed === true && + result?.preflightFailure?.kind === "transport" && + result?.error?.includes(postFenceFailure.message) === true && + stagingTree.length === 0 && + finalExists; + record( + "gen4-c3-post-fence-terminal-ledger", + "AC13", + rootCommand, + observed, + pass, + "A post-fence failure triggered the pre-fence cleanup downgrade, advanced to another candidate, or lost terminal ledger evidence.", + ); + expect(pass).toBe(true); + }); +}); + +describe("autorouting boundary red-team generation 5 delta re-attacks", () => { + it("C4 terminal disposition fails closed and the candidate ledger never exceeds three unique selectors", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen5-disposition-")); + const models = ["one", "two", "three", "four", "five"].map(id => + model("test", id, true, { + thinking: { mode: "effort", minLevel: "minimal", maxLevel: "high" }, + }), + ); + const registry = { + authStorage: {}, + getAll: () => models, + getAvailable: () => models, + getApiKey: async () => "key", + } as never; + const optionsFor = (id: string, candidates: string[]): Parameters[0] => ({ + cwd: root, + agent: taskAgent, + task: id, + assignment: id, + index: 0, + id, + modelOverride: candidates, + settings: Settings.isolated(), + modelRegistry: registry, + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: candidates, + autoroutingSkips: [], + routing: { + tier: "fast", + requestedSelector: candidates[0] ?? "test/one", + substitutions: [], + }, + sessionFile: path.join(root, `${id}.jsonl`), + }); + const terminal = Object.assign(new Error("typed terminal transport"), { + transportFailure: { kind: "transport" as const, status: 418 }, + }); + let terminalCalls = 0; + const terminalSpy = vi.spyOn(sdkModule, "createAgentSession").mockImplementation(async () => { + terminalCalls++; + throw terminal; + }); + let terminalResult: Awaited> | undefined; + try { + terminalResult = await runSubprocess( + optionsFor("gen5-terminal", ["test/one", "test/two", "test/three", "test/four"]), + ); + } finally { + terminalSpy.mockRestore(); + } + const transient = Object.assign(new Error("bounded transient"), { transient: true }); + let budgetCalls = 0; + const budgetSpy = vi.spyOn(sdkModule, "createAgentSession").mockImplementation(async () => { + budgetCalls++; + throw transient; + }); + let budgetResult: Awaited> | undefined; + try { + budgetResult = await runSubprocess( + optionsFor("gen5-budget", ["test/one", "test/one", "test/two", "test/three", "test/four"]), + ); + } finally { + budgetSpy.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + const budgetCandidates = budgetResult?.routing?.attempts?.map(attempt => attempt.selector) ?? []; + const observed = { + terminalCalls, + terminalAttempts: terminalResult?.routing?.attempts, + terminalFailure: terminalResult?.preflightFailure, + terminal: terminalResult?.routing?.terminal, + budgetCalls, + budgetAttempts: budgetResult?.routing?.attempts, + budget: budgetResult?.routing?.terminal, + budgetCandidates, + }; + const pass = + terminalCalls === 1 && + JSON.stringify(terminalResult?.routing?.attempts) === + JSON.stringify([{ selector: "test/one", phase: "probe", code: "unclassified_terminal" }]) && + terminalResult?.routing?.terminal === "preflight_exhausted" && + budgetCalls === 3 && + budgetResult?.routing?.attempts?.length === 3 && + new Set(budgetCandidates).size === 3 && + !budgetCandidates.includes("test/four") && + budgetResult?.routing?.attempts?.every(attempt => attempt.code === "spawn_transient_retry") === true; + record( + "gen5-disposition-and-budget", + "AC13", + `${rootCommand} -t gen5-disposition-and-budget`, + observed, + pass, + "A terminal classification advanced the ledger, or preflight consumed more than three unique candidates.", + ); + expect(pass).toBe(true); + }); + + it("C5 terminal diagnostics retain the original failure while stripping controls, normalizing, and capping", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen5-diagnostic-")); + const models = [model("test", "diagnostic", true)]; + const registry = { + authStorage: {}, + getAll: () => models, + getAvailable: () => models, + getApiKey: async () => "key", + } as never; + const marker = "GEN5-ORIGINAL-FAILURE"; + const hostileMessage = `${marker}\u0000\u001b[31m\u0007\r\n${"A".repeat(900)}\u001b[0m`; + const hostile = Object.assign(new Error(hostileMessage), { transient: false }); + const createSpy = vi.spyOn(sdkModule, "createAgentSession").mockRejectedValue(hostile); + let result: Awaited> | undefined; + try { + result = await runSubprocess({ + cwd: root, + agent: taskAgent, + task: "gen5 diagnostic", + assignment: "gen5 diagnostic", + index: 0, + id: "gen5-diagnostic", + modelOverride: ["test/diagnostic"], + settings: Settings.isolated(), + modelRegistry: registry, + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: ["test/diagnostic"], + autoroutingSkips: [], + routing: { + tier: "fast", + requestedSelector: "test/diagnostic", + substitutions: [], + }, + sessionFile: path.join(root, "diagnostic.jsonl"), + }); + } finally { + createSpy.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + const prefix = "Last candidate diagnostic: "; + const stderrDiagnostic = result?.stderr.split(prefix)[1] ?? ""; + const errorDiagnostic = result?.error?.split(prefix)[1] ?? ""; + const summaryDiagnostic = result?.setupFailure?.summary.split(prefix)[1] ?? ""; + const observed = { + stderr: result?.stderr, + error: result?.error, + setupFailure: result?.setupFailure, + stderrDiagnostic, + errorDiagnostic, + summaryDiagnostic, + stderrLength: stderrDiagnostic.length, + containsMarker: stderrDiagnostic.includes(marker), + containsAnsiControl: /\u001b/.test(stderrDiagnostic), + containsLineBreak: /[\r\n]/.test(stderrDiagnostic), + containsNfkc: stderrDiagnostic.includes("A"), + }; + const pass = + result?.routing?.terminal === "preflight_exhausted" && + // Separator is a space, not "\n": the diagnostic is rendered on a single line so a hostile + // message can never forge an extra log line. Bounding/redaction is delegated to + // createSetupFailureSummary (the established egress sanitizer, which also redacts + // credentials and absolute paths), so the cap is its cap rather than a local 512 constant. + result?.stderr?.startsWith("Autorouting preflight exhausted. Last candidate diagnostic: ") === true && + result?.error === result?.stderr && + result?.setupFailure?.summary.includes(prefix) === true && + stderrDiagnostic.length > 0 && + stderrDiagnostic.length <= 512 && + stderrDiagnostic.includes(marker) && + !/[\u0000-\u001f\u007f-\u009f]/.test(stderrDiagnostic) && + !/[\r\n]/.test(stderrDiagnostic) && + stderrDiagnostic.includes("A") && + errorDiagnostic === stderrDiagnostic && + summaryDiagnostic === stderrDiagnostic; + record( + "gen5-terminal-diagnostic-bounds", + "AC13", + `${rootCommand} -t gen5-terminal-diagnostic-bounds`, + observed, + pass, + "Terminalization dropped the original failure or allowed control/newline injection beyond the 512-character diagnostic bound.", + ); + expect(pass).toBe(true); + }); + + it("C6 URI re-keying handles scheme case, zero-padded IDs, malformed artifact tails, and opaque query/fragment payloads", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen5-uri-grammar-")); + const manager = await SessionManager.openStaged( + path.join(root, "candidate.jsonl"), + undefined, + "gen5-uri-grammar", + ); + const original = { + uppercaseArtifact: "ARTIFACT://3:1-1", + uppercaseAgent: "Agent://3:raw", + leadingZero: "artifact://007:1-1", + overlongTail: "artifact://3:raw:1-100:extra", + adjacent: "artifact://3:1-1artifact://4:1-1", + queryNested: "artifact://3?next=artifact://4:1-1", + fragmentNested: "artifact://3#next=agent://4:raw", + queryIdOnly: "prefix?artifact://4:1-1", + plainWhitespace: "artifact://3:1-1 next artifact://4:1-1", + }; + manager.appendCustomEntry("gen5-uri-grammar", original); + const remap = new Map([ + ["3", "30"], + ["4", "40"], + ["007", "70"], + ]); + const parserObserved = { + uppercaseArtifact: splitInternalUrlSel(original.uppercaseArtifact), + uppercaseAgent: splitInternalUrlSel(original.uppercaseAgent), + leadingZero: splitInternalUrlSel(original.leadingZero), + overlongTail: splitInternalUrlSel(original.overlongTail), + adjacent: splitInternalUrlSel(original.adjacent), + }; + await manager.remapStagedArtifactReferences(remap); + const stagedFile = manager.getSessionFile(); + const stagedText = stagedFile ? await Bun.file(stagedFile).text() : ""; + const entry = stagedText + .split("\n") + .filter(Boolean) + .map(line => JSON.parse(line) as { customType?: string; data?: typeof original }) + .find(item => item.customType === "gen5-uri-grammar"); + await manager.discardStaged(); + await rm(root, { recursive: true, force: true }); + const remapped = entry?.data; + const observed = { parserObserved, remapped, stagedText }; + const pass = + remapped?.uppercaseArtifact === "ARTIFACT://30:1-1" && + remapped?.uppercaseAgent === "Agent://30:raw" && + remapped?.leadingZero === "artifact://70:1-1" && + remapped?.overlongTail === "artifact://30:raw:1-100:extra" && + remapped?.adjacent === "artifact://30:1-1artifact://4:1-1" && + remapped?.queryNested === "artifact://30?next=artifact://4:1-1" && + remapped?.fragmentNested === "artifact://30#next=agent://4:raw" && + remapped?.queryIdOnly === "prefix?artifact://4:1-1" && + remapped?.plainWhitespace === "artifact://3:1-1 next artifact://4:1-1"; + record( + "gen5-uri-rekey-grammar", + "AC13", + `${rootCommand} -t gen5-uri-rekey-grammar`, + observed, + pass, + "URI re-keying diverged from embedded-token grammar: uppercase schemes were missed or nested query/fragment IDs cascaded into remapping.", + ); + expect(pass).toBe(true); + }); +}); +describe("C7 disposition oracle for auth resolution and post-fence transport", () => { + it("keeps auth resolution advancing while transport failures stay terminal after the fence", () => { + const auth = classifyAutoroutingPreflightFailure( + Object.assign(new Error("auth unavailable"), { transient: false, credentialMissing: true }), + "auth_resolve", + ); + const transientSession = classifyAutoroutingPreflightFailure( + Object.assign(new Error("bootstrap transient"), { transient: true }), + "tool_bootstrap", + ); + const postFence = classifyAutoroutingPreflightFailure( + { transportFailure: { kind: "transport" as const, status: 503 } }, + "session_open", + ); + // An unmarked exception surfacing while op is auth_resolve must NOT be treated as the + // deliberate missing-credential signal: an unexpected keychain/config error must fail + // closed (terminal), not silently advance as if credentials were simply absent. + const unexpectedAuthError = classifyAutoroutingPreflightFailure( + Object.assign(new Error("keychain access denied"), { transient: false }), + "auth_resolve", + ); + const observed = { auth, transientSession, postFence, unexpectedAuthError }; + const pass = + auth.kind === "local" && + auth.op === "auth_resolve" && + transientSession.kind === "local" && + transientSession.transient === true && + postFence.kind === "transport" && + postFence.class === "server" && + unexpectedAuthError.kind === "local" && + unexpectedAuthError.op !== "auth_resolve" && + unexpectedAuthError.transient === false; + record( + "gen5-fence-disposition", + "AC13", + `${rootCommand} -t gen5-fence-disposition`, + observed, + pass, + "Auth resolution or transient bootstrap classification changed, typed post-fence transport facts were not preserved, or an unmarked auth_resolve exception was wrongly treated as the deliberate missing-credential signal.", + ); + expect(pass).toBe(true); + }); +}); + +describe("autorouting boundary red-team generation 6 varied delta re-attacks", () => { + it("C8 diagnostic egress redacts secrets, paths, Unicode separators, and long multibyte payloads", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen6-diagnostic-")); + const registry = { + authStorage: {}, + getAll: () => [model("test", "diagnostic")], + getAvailable: () => [model("test", "diagnostic")], + getApiKey: async () => "key", + } as never; + const secretMessage = [ + "Authorization: Bearer super-secret-token", + "Cookie: session=secret-cookie", + "x-api-key: x-api-secret-value", + "https://alice:password@example.invalid/api", + "api_key=sk-live-secret-value", + "token=provider-secret-token", + "sk-proj-abcdefghijklmnopqrstuvwxyz123456", + "https://example.invalid/?auth=AKIAIOSFODNN7EXAMPLE", + `file://${process.env.HOME ?? "/Users/secret"}/private/config.yml`, + "/Users/secret/private.txt", + "/private/credentials.json", + "C:\\Users\\secret\\.ssh\\id_rsa", + "$HOME/.ssh/id_rsa", + `ansi=\u001b[31mred\u001b[0m osc=\u001b]8;;https://secret.invalid\u0007link\u001b]8;;\u0007`, + `line${"\u2028"}next${"\u2029"}tail\rline\vvertical\ffeed ${"界".repeat(1200)}`, + ].join(" "); + const failure = Object.assign(new Error(secretMessage), { transient: false }); + const createSpy = vi.spyOn(sdkModule, "createAgentSession").mockRejectedValue(failure); + let result: Awaited> | undefined; + try { + result = await runSubprocess({ + cwd: root, + agent: taskAgent, + task: "gen6 diagnostic", + assignment: "gen6 diagnostic", + index: 0, + id: "gen6-diagnostic", + modelOverride: ["test/diagnostic"], + settings: Settings.isolated(), + modelRegistry: registry, + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: ["test/diagnostic"], + autoroutingSkips: [], + routing: { tier: "fast", requestedSelector: "test/diagnostic", substitutions: [] }, + sessionFile: path.join(root, "diagnostic.jsonl"), + }); + } finally { + createSpy.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + const extract = (value: string): string => value.split("Last candidate diagnostic: ")[1] ?? ""; + const diagnostic = extract(result?.stderr ?? ""); + const renderings = [ + extract(result?.stderr ?? ""), + extract(result?.error ?? ""), + extract(result?.setupFailure?.summary ?? ""), + ]; + const observed = { + diagnostic, + length: diagnostic.length, + containsBearerSecret: diagnostic.includes("super-secret-token"), + containsCookieSecret: diagnostic.includes("secret-cookie"), + containsPassword: diagnostic.includes("password@"), + containsApiKey: diagnostic.includes("sk-live-secret-value") || diagnostic.includes("x-api-secret-value"), + containsProviderToken: + diagnostic.includes("provider-secret-token") || + diagnostic.includes("sk-proj-abcdefghijklmnopqrstuvwxyz123456") || + diagnostic.includes("AKIAIOSFODNN7EXAMPLE"), + containsAbsolutePath: + diagnostic.includes("/Users/secret") || + diagnostic.includes("/private/credentials.json") || + diagnostic.includes("C:\\Users\\secret"), + containsHomePath: diagnostic.includes("$HOME/") || diagnostic.includes("id_rsa"), + containsControl: /[\u0000-\u001f\u007f-\u009f\u001b]/.test(diagnostic), + containsSeparators: /[\r\n\u000b\u2028\u2029]/.test(diagnostic), + }; + const pass = + result?.routing?.terminal === "preflight_exhausted" && + diagnostic.length > 0 && + diagnostic.length <= 512 && + !observed.containsBearerSecret && + !observed.containsCookieSecret && + !observed.containsPassword && + !observed.containsApiKey && + !observed.containsProviderToken && + !observed.containsAbsolutePath && + !observed.containsHomePath && + !observed.containsControl && + !observed.containsSeparators && + renderings.every(value => value === diagnostic); + record( + "gen6-diagnostic-redaction-unicode", + "AC13", + `${rootCommand} -t gen6-diagnostic-redaction-unicode`, + observed, + pass, + "Sanitized terminal diagnostics leaked credentials, paths, Unicode line separators, or diverged across receipt renderings.", + ); + expect(pass).toBe(true); + }); + it("C9 whole-token URI shapes preserve opaque tails and remap without cascade or double mapping", async () => { + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen6-uri-shapes-")); + const manager = await SessionManager.openStaged(path.join(root, "candidate.jsonl"), undefined, "gen6-uri-shapes"); + const original = { + compound: "ArTiFaCt://12:raw:1-20", + plusTail: "agent://12:+1-2", + minusTail: "agent://12:-1", + percentId: "artifact://%31%32:raw", + query: "artifact://12?ref=artifact://13:raw", + fragment: "Agent://12#ref=artifact://13:raw", + cycle: "artifact://12:raw", + prose: "prefix artifact://12:raw suffix", + adjacent: "artifact://12:rawartifact://13:raw", + }; + manager.appendCustomEntry("gen6-uri-shapes", original); + const expected = structuredClone(original); + const map = new Map([ + ["12", "13"], + ["13", "12"], + ]); + await manager.remapStagedArtifactReferences(map); + const file = manager.getSessionFile(); + const text = file ? await Bun.file(file).text() : ""; + const entry = text + .split("\n") + .filter(Boolean) + .map(line => JSON.parse(line) as { customType?: string; data?: typeof original }) + .find(item => item.customType === "gen6-uri-shapes"); + await manager.discardStaged(); + await rm(root, { recursive: true, force: true }); + const remapped = entry?.data; + const observed = { + original, + expected, + map: [...map.entries()], + remapped, + parser: splitInternalUrlSel(expected.compound), + }; + const pass = + remapped?.compound === "ArTiFaCt://13:raw:1-20" && + remapped?.plusTail === expected.plusTail && + remapped?.minusTail === expected.minusTail && + remapped?.percentId === expected.percentId && + remapped?.query === "artifact://13?ref=artifact://13:raw" && + remapped?.fragment === "Agent://13#ref=artifact://13:raw" && + remapped?.cycle === "artifact://13:raw" && + remapped?.prose === expected.prose && + remapped?.adjacent === "artifact://13:rawartifact://13:raw"; + record( + "gen6-uri-whole-token-shapes", + "AC13", + `${rootCommand} -t gen6-uri-whole-token-shapes`, + observed, + pass, + "Whole-token URI re-keying cascaded, decoded percent IDs, rewrote opaque query/fragment payloads, or rewrote prose/adjacent tokens.", + ); + expect(pass).toBe(true); + }); +}); + +describe("autorouting boundary red-team generation 8 delta re-attacks", () => { + it("CLEAN ownership keeps sibling residue while removing the artifact's own native and quarantine residue", async () => { + const root = await fs.realpath(await mkdtemp(path.join(tmpdir(), "autorouting-gen8-cleanup-ownership-"))); + const parentDir = path.join(root, "parent"); + const parent = new ArtifactManager(new ManagedSessionDescendantStore(managedDirectoryRoot(root), parentDir)); + await parent.save("sibling", "tool"); + const filename = "1.tool.log"; + await parent.publishNamedNoReplace(filename, Buffer.from("target", "utf8")); + const siblingQuarantine = path.join(parentDir, "11.tool.log.removing"); + const siblingPlaceholder = path.join(parentDir, ".gjc-exact-unlink-placeholder-other"); + const ownNativePlaceholder = path.join(parentDir, ".gjc-exact-unlink-placeholder-owned"); + const ownQuarantine = path.join(parentDir, `${filename}.removing`); + await fs.writeFile(siblingQuarantine, "sibling quarantine", "utf8"); + await fs.writeFile(siblingPlaceholder, "sibling placeholder", "utf8"); + const usesRetainedAuthority = process.platform === "linux"; + const originalExactUnlink = native.exactUnlink; + const exactUnlinkSpy = usesRetainedAuthority + ? undefined + : vi.spyOn(native, "exactUnlink").mockImplementation((pathname, identity) => { + if (pathname !== path.join(parentDir, filename)) return originalExactUnlink(pathname, identity); + fsSync.unlinkSync(pathname); + fsSync.writeFileSync(ownNativePlaceholder, "owned native residue", "utf8"); + fsSync.writeFileSync(ownQuarantine, "owned quarantine residue", "utf8"); + return { ok: true }; + }); + let removed = false; + try { + removed = await parent.removeNamedBestEffort(filename); + } finally { + exactUnlinkSpy?.mockRestore(); + } + const remaining = await tree(parentDir); + const observed = { + removed, + remaining, + targetBytes: await fileBytes(path.join(parentDir, filename)), + siblingQuarantineBytes: await fileBytes(siblingQuarantine), + siblingPlaceholderBytes: await fileBytes(siblingPlaceholder), + ownNativePlaceholderBytes: await fileBytes(ownNativePlaceholder), + ownQuarantineBytes: await fileBytes(ownQuarantine), + }; + const pass = + removed && + !remaining.includes(filename) && + remaining.includes("0.tool.log") && + remaining.includes(path.basename(siblingQuarantine)) && + remaining.includes(path.basename(siblingPlaceholder)) && + (usesRetainedAuthority || !remaining.includes(path.basename(ownNativePlaceholder))) && + (usesRetainedAuthority || !remaining.includes(path.basename(ownQuarantine))) && + observed.siblingQuarantineBytes !== null && + observed.siblingPlaceholderBytes !== null && + observed.targetBytes === null; + record( + "gen8-cleanup-ownership-boundary", + "AC13", + rootCommand, + observed, + pass, + "Owned cleanup residue was not removed, or a sibling .removing/native placeholder was cross-deleted by a substring match.", + ); + expect(pass).toBe(true); + await rm(root, { recursive: true, force: true }); + }); + + it("CLEAN publication rollback retires a failed reserve block and preserves the original publication error", async () => { + const root = await fs.realpath(await mkdtemp(path.join(tmpdir(), "autorouting-gen8-publication-retire-"))); + const parentDir = path.join(root, "parent"); + const parent = new ArtifactManager(new ManagedSessionDescendantStore(managedDirectoryRoot(root), parentDir)); + await parent.save("sibling", "tool"); + const staged = parent.createAttemptStaging("gen8-publication-retire"); + await staged.save("first", "tool"); + await staged.save("second", "tool"); + const parentStore = parent.getManagedStore(); + if (!parentStore) throw new Error("managed parent store unavailable"); + const publicationError = new Error("gen8-publication-failure"); + const removalAttempts: string[] = []; + let publishCalls = 0; + const realPublish = parentStore.publishNoReplace.bind(parentStore); + const publishSpy = vi.spyOn(parentStore, "publishNoReplace").mockImplementation(async (filename, bytes) => { + publishCalls++; + if (publishCalls === 2) throw publicationError; + await realPublish(filename, bytes); + }); + const removeSpy = vi.spyOn(parent, "removeNamedBestEffort").mockImplementation(async filename => { + removalAttempts.push(filename); + return false; + }); + let failure: unknown; + try { + await parent.commitAttemptStaging(staged, "gen8-publication-retire"); + } catch (error) { + failure = error; + } finally { + publishSpy.mockRestore(); + removeSpy.mockRestore(); + } + await staged.discardAttemptStaging(); + const errors = failure instanceof AggregateError ? failure.errors : []; + const nextIds = [parent.allocateId(), parent.allocateId()]; + const observed = { + failure: failure instanceof Error ? { name: failure.name, message: failure.message } : String(failure), + errors: errors.map(error => (error instanceof Error ? error.message : String(error))), + originalErrorPreserved: errors[0] === publicationError, + publishCalls, + removalAttempts, + leakedFirst: await parent.exists("1"), + leakedSecond: await parent.exists("2"), + nextIds, + allocatedIds: parent.getAllocatedIds(), + }; + const pass = + failure instanceof AggregateError && + errors.length >= 2 && + errors[0] === publicationError && + errors.some(error => String(error).includes("1.tool.log")) && + publishCalls === 2 && + removalAttempts.length === 1 && + removalAttempts[0] === "1.tool.log" && + observed.leakedFirst && + !observed.leakedSecond && + JSON.stringify(nextIds) === JSON.stringify([3, 4]) && + !nextIds.includes(1) && + !nextIds.includes(2); + record( + "gen8-publication-rollback-retires-reserve-block", + "AC13", + rootCommand, + observed, + pass, + "A failed publication rollback hid the original error, rewound a block containing a leaked artifact, or reallocated a retired ID.", + ); + expect(pass).toBe(true); + await rm(root, { recursive: true, force: true }); + }); + + it("CLEAN rollbackLastAttemptCommit retires failed removals while a successful rollback still rewinds the tail", async () => { + const root = await fs.realpath(await mkdtemp(path.join(tmpdir(), "autorouting-gen8-rollback-retire-"))); + const parentDir = path.join(root, "parent"); + const parent = new ArtifactManager(new ManagedSessionDescendantStore(managedDirectoryRoot(root), parentDir)); + await parent.save("sibling", "tool"); + const staged = parent.createAttemptStaging("gen8-rollback-retire"); + await staged.save("first", "tool"); + await staged.save("second", "tool"); + const mapping = await parent.commitAttemptStaging(staged, "gen8-rollback-retire"); + const removalAttempts: string[] = []; + const removeSpy = vi.spyOn(parent, "removeNamedBestEffort").mockImplementation(async filename => { + removalAttempts.push(filename); + return false; + }); + let rollbackFailure: unknown; + try { + await parent.rollbackLastAttemptCommit("gen8-rollback-retire"); + } catch (error) { + rollbackFailure = error; + } finally { + removeSpy.mockRestore(); + } + const nextIds = [parent.allocateId(), parent.allocateId()]; + + const successRoot = await fs.realpath(await mkdtemp(path.join(tmpdir(), "autorouting-gen8-rollback-success-"))); + const successDir = path.join(successRoot, "parent"); + const successParent = new ArtifactManager( + new ManagedSessionDescendantStore(managedDirectoryRoot(successRoot), successDir), + ); + await successParent.save("sibling", "tool"); + const successStaged = successParent.createAttemptStaging("gen8-rollback-success"); + await successStaged.save("candidate", "tool"); + const successMapping = await successParent.commitAttemptStaging(successStaged, "gen8-rollback-success"); + await successParent.rollbackLastAttemptCommit("gen8-rollback-success"); + const rewoundId = successParent.allocateId(); + const successPublishedId = successMapping.get("0") ?? "missing"; + const observed = { + mapping: [...mapping.entries()], + rollbackFailure: + rollbackFailure instanceof Error + ? { name: rollbackFailure.name, message: rollbackFailure.message } + : String(rollbackFailure), + removalAttempts, + leakedIds: { + one: await parent.exists("1"), + two: await parent.exists("2"), + }, + nextIds, + successMapping: [...successMapping.entries()], + rewoundId, + successPublishedId, + successPublishedStillExists: await successParent.exists(successPublishedId), + }; + const pass = + rollbackFailure instanceof Error && + rollbackFailure.message.includes("1.tool.log") && + removalAttempts.length === 2 && + new Set(removalAttempts).size === 2 && + observed.leakedIds.one && + observed.leakedIds.two && + JSON.stringify(nextIds) === JSON.stringify([3, 4]) && + successMapping.get("0") === "1" && + rewoundId === 1 && + !observed.successPublishedStillExists; + record( + "gen8-rollback-last-attempt-retirement-and-tail-rewind", + "AC13", + rootCommand, + observed, + pass, + "rollbackLastAttemptCommit failed to retire the reserved block, reused leaked IDs, or stopped rewinding a fully successful tail rollback.", + ); + expect(pass).toBe(true); + await rm(root, { recursive: true, force: true }); + await rm(successRoot, { recursive: true, force: true }); + }); + + it("C9 the live routed selector reaches session creation byte-exact, unaffected by evidence bounding", async () => { + // boundedSelector (NFKC-normalize, strip control chars, truncate to 256) exists to bound + // text that is rendered or persisted as evidence/telemetry. It must never be applied to the + // selector actually used to resolve and execute the model: candidates were already validated + // against the live snapshot by normalizeTierSelector, so re-transforming the live selector + // could compose characters differently or truncate a long-but-valid id, sending execution to + // a model that never passed preflight. Use an id long enough that boundedSelector's 256-char + // cap would corrupt it if applied to the live selector, and confirm the exact byte-for-byte + // selector reaches createAgentSession's resolved model. + const root = await mkdtemp(path.join(tmpdir(), "autorouting-gen9-live-selector-")); + const finalPath = path.join(root, "candidate.jsonl"); + const longId = `over-256-chars-${"x".repeat(280)}`; + const longSelector = `test/${longId}`; + const models = [ + model("test", longId, true, { + headers: {}, + compat: {}, + thinking: { mode: "effort", minLevel: "minimal", maxLevel: "high" }, + }), + ]; + const authStorage = await AuthStorage.create(":memory:"); + const modelRegistry = new ModelRegistry(authStorage); + vi.spyOn(modelRegistry, "getAll").mockReturnValue(models); + vi.spyOn(modelRegistry, "getAvailable").mockReturnValue(models); + vi.spyOn(modelRegistry, "getApiKey").mockImplementation(async () => "key"); + const originalCreate = sdkModule.createAgentSession; + let capturedModelId: string | undefined; + const createSpy = vi.spyOn(sdkModule, "createAgentSession").mockImplementation(async options => { + capturedModelId = options?.model?.id; + const result = await originalCreate(options); + vi.spyOn(result.session, "prompt").mockImplementation(async (_message, promptOptions) => { + if (promptOptions?.onPreflightAcceptCommit) await promptOptions.onPreflightAcceptCommit(); + else promptOptions?.onPreflightAccepted?.(); + result.session.agent.emitExternalEvent({ + type: "tool_execution_end", + toolCallId: "gen9-yield", + toolName: "yield", + result: { content: [], details: { status: "success", data: {} } }, + isError: false, + } as never); + }); + vi.spyOn(result.session, "waitForIdle").mockResolvedValue(undefined); + return result; + }); + let result: Awaited> | undefined; + try { + result = await runSubprocess({ + cwd: root, + agent: taskAgent, + task: "gen9 live selector", + assignment: "gen9 live selector", + index: 0, + id: "gen9-live-selector", + modelOverride: [longSelector], + settings: Settings.isolated(), + modelRegistry, + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: [longSelector], + autoroutingSkips: [], + routing: { + tier: "fast", + // The evidence-side requestedSelector/effectiveModel are legitimately bounded to + // <=256 chars by assertRoutingEvidenceInvariant; that bounding is correct and not + // under test here. Only the *live* selector must reach execution untransformed. + requestedSelector: "test/short-placeholder", + effectiveModel: "test/short-placeholder", + substitutions: [], + }, + sessionFile: finalPath, + }); + } finally { + createSpy.mockRestore(); + authStorage.close(); + await rm(root, { recursive: true, force: true }); + } + const observed = { + capturedModelId, + expectedModelId: longId, + capturedModelIdLength: capturedModelId?.length, + exitCode: result?.exitCode, + preflightFenceCrossed: result?.preflightFenceCrossed, + attempts: result?.routing?.attempts, + }; + const pass = + capturedModelId === longId && + (capturedModelId?.length ?? 0) > 256 && + result?.exitCode === 0 && + result?.preflightFenceCrossed === true; + record( + "gen9-c9-live-selector-untransformed", + "AC13", + rootCommand, + observed, + pass, + "The live routed selector was truncated or otherwise transformed by evidence-bounding logic before reaching session creation, causing execution to diverge from the preflight-validated candidate.", + ); + expect(pass).toBe(true); + }); +}); diff --git a/packages/coding-agent/test/autorouting-generator.test.ts b/packages/coding-agent/test/autorouting-generator.test.ts new file mode 100644 index 0000000000..b39427ae12 --- /dev/null +++ b/packages/coding-agent/test/autorouting-generator.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from "bun:test"; +import type { Model } from "@gajae-code/ai"; +import { resolveTaskRouting } from "../src/config/autorouting"; +import { validateAutoroutingEffective } from "../src/config/autorouting-contract"; +import { canonicalJsonBytes, generateTierChains } from "../src/config/autorouting-generator"; +import type { CuratedTierLabels } from "../src/config/autorouting-tier-map"; +import { projectCatalogProviderOrder } from "../src/config/provider-selection-policy"; + +function model(provider: string, id: string, reasoning = true): Model { + return { + provider, + id, + name: id, + api: "openai-completions", + baseUrl: "https://example.invalid", + reasoning, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4096, + }; +} + +function bytes(value: unknown): string { + return new TextDecoder().decode(canonicalJsonBytes(value)); +} + +const syntheticLabels = { + "alpha/fast": [{ tier: "fast", rank: 1 }], + "alpha/slow": [{ tier: "fast", rank: 2 }], + "beta/fast": [{ tier: "fast", rank: 1 }], + "gamma/fast": [{ tier: "fast", rank: 1 }], + "alpha/strong": [{ tier: "strong", effort: "high", rank: 1 }], +} satisfies CuratedTierLabels; + +const syntheticMap = { labels: syntheticLabels, skips: {}, version: 1 }; + +describe("autorouting generator", () => { + it("is byte-identical across repeated runs", () => { + const catalog = [model("alpha", "fast"), model("alpha", "slow"), model("alpha", "strong")]; + const setup = { schema: 1 as const, providers: ["alpha"] }; + const first = generateTierChains(setup, syntheticMap, catalog); + const second = generateTierChains(setup, syntheticMap, catalog); + expect(bytes(first.tiers)).toBe(bytes(second.tiers)); + expect(first).toEqual(second); + }); + + it("rejects generated selectors that exceed the runtime contract", () => { + const id = "x".repeat(251); + const labels = { [`alpha/${id}`]: [{ tier: "fast" as const, rank: 1 }] } satisfies CuratedTierLabels; + expect(() => + generateTierChains({ schema: 1, providers: ["alpha"] }, { labels, skips: {}, version: 1 }, [ + model("alpha", id), + ]), + ).toThrow(/length bound/); + }); + + it("orders by provider declaration, then curation rank, and is stable under catalog permutation", () => { + const catalog = [model("alpha", "slow"), model("gamma", "fast"), model("beta", "fast"), model("alpha", "fast")]; + const setup = { schema: 1 as const, providers: ["beta", "alpha", "gamma"] }; + const first = generateTierChains(setup, syntheticMap, catalog); + const second = generateTierChains(setup, syntheticMap, [...catalog].reverse()); + expect(first.tiers.fast).toEqual(["beta/fast", "alpha/fast", "alpha/slow", "gamma/fast"]); + expect(bytes(first.tiers)).toBe(bytes(second.tiers)); + }); + + it("omits unlabeled and empty tiers so downstream routing falls through honestly", () => { + const result = generateTierChains( + { schema: 1, providers: ["qianfan"] }, + { labels: { "qianfan/only": [{ tier: "fast", rank: 1 }] }, skips: {}, version: 1 }, + [model("qianfan", "only", false)], + ); + expect(result.tiers).toEqual({ fast: ["qianfan/only"] }); + const routing = resolveTaskRouting({ + effectiveAutorouting: validateAutoroutingEffective({ enabled: true, tiers: result.tiers }), + requestedTier: "balanced", + availableModels: [model("qianfan", "only", false)], + }); + expect(routing).toMatchObject({ kind: "manual-fallback", reason: "tier_missing_in_map" }); + }); + + it("uses allowlists only as eligibility filters and never as a priority channel", () => { + const catalog = [model("alpha", "fast"), model("alpha", "slow"), model("beta", "fast")]; + const setup = { + schema: 1 as const, + providers: ["alpha", "beta"], + models: ["alpha/slow", "beta/fast", "alpha/fast"], + }; + const result = generateTierChains(setup, syntheticMap, catalog); + expect(result.tiers.fast).toEqual(["alpha/fast", "alpha/slow", "beta/fast"]); + }); + + it("matches setup provider casing against catalog keys case-insensitively", () => { + const catalog = [model("OpenAI", "gpt-test"), model("openai", "gpt-other")]; + const result = generateTierChains( + { schema: 1, providers: ["OpenAI"] }, + { + labels: { + "openai/gpt-test": [{ tier: "fast", rank: 1 }], + "openai/gpt-other": [{ tier: "fast", rank: 2 }], + }, + skips: {}, + version: 1, + }, + catalog, + ); + // Selectors keep catalog spelling; only matching is case-insensitive. + expect(result.tiers.fast).toEqual(["OpenAI/gpt-test", "openai/gpt-other"]); + }); + + it("de-duplicates setup providers that differ only by case", () => { + const catalog = [model("alpha", "fast"), model("alpha", "strong")]; + const result = generateTierChains({ schema: 1, providers: ["Alpha", "alpha", "ALPHA"] }, syntheticMap, catalog); + expect(result.tiers.fast).toEqual(["alpha/fast"]); + }); + + it("matches allowlist selectors case-insensitively", () => { + const catalog = [model("alpha", "fast"), model("alpha", "slow"), model("alpha", "strong")]; + const result = generateTierChains( + { schema: 1, providers: ["alpha"], models: ["ALPHA/slow"] }, + syntheticMap, + catalog, + ); + expect(result.tiers.fast).toEqual(["alpha/slow"]); + }); + + it("deduplicates duplicate catalog overlays without inventing tier membership", () => { + const catalog = [model("alpha", "fast"), model("alpha", "fast"), model("alpha", "strong")]; + const result = generateTierChains({ schema: 1, providers: ["alpha", "alpha"] }, syntheticMap, catalog); + expect(result.tiers.fast).toEqual(["alpha/fast"]); + expect(result.tiers.strong).toEqual(["alpha/strong:high"]); + }); + + it("does not consult credentials or auth state", () => { + const setup = { schema: 1 as const, providers: ["alpha"] }; + const catalog = [model("alpha", "fast")]; + const authenticated = generateTierChains(setup, syntheticMap, catalog); + const unauthenticated = generateTierChains( + setup, + syntheticMap, + catalog.map(entry => ({ ...entry, baseUrl: "https://other.invalid" })), + ); + expect(bytes(authenticated.tiers)).toBe(bytes(unauthenticated.tiers)); + expect(authenticated.declarationFingerprint).toBe(unauthenticated.declarationFingerprint); + }); + + it("matches the frozen canonical-byte golden fixtures", async () => { + const fixtureNames = ["anthropic", "anthropic-google", "openai", "thin-single-provider"] as const; + for (const name of fixtureNames) { + const fixture = (await Bun.file(new URL(`./autorouting-golden/${name}.json`, import.meta.url)).json()) as { + catalog: Array<{ provider: string; id: string; reasoning: boolean }>; + setup: { schema: 1; providers: string[] }; + expectedTiers: Record; + expectedCanonicalBytes: string; + }; + const catalog = fixture.catalog.map(entry => model(entry.provider, entry.id, entry.reasoning)); + const result = generateTierChains(fixture.setup, undefined, catalog); + expect(bytes(result.tiers)).toBe(fixture.expectedCanonicalBytes); + expect(result.tiers).toEqual(fixture.expectedTiers); + } + }); + + it("derives the declaration from provider priority before generating tiers", async () => { + // The other fixtures hand the generator an already-sorted setup, so they never + // exercise the derivation. This one starts from configured order plus catalog + // and runs the real projection end to end. + const fixture = (await Bun.file( + new URL("./autorouting-golden/policy-derived-provider-order.json", import.meta.url), + ).json()) as { + catalog: Array<{ provider: string; id: string; reasoning: boolean }>; + configuredProviderOrder: string[]; + expectedProviderOrder: string[]; + setup: { schema: 1; providers: string[] }; + expectedTiers: Record; + expectedCanonicalBytes: string; + }; + const catalog = fixture.catalog.map(entry => model(entry.provider, entry.id, entry.reasoning)); + // Call the shipped projection, not a copy of it: this is the same function + // ModelRegistry.autoroutingProviderOrder() delegates to, so breaking it breaks + // this golden. + const derived = projectCatalogProviderOrder(fixture.configuredProviderOrder, catalog); + + // A configured provider absent from the catalog must not survive into the setup. + expect(fixture.configuredProviderOrder).toContain("ghost-provider"); + expect(derived).toEqual(fixture.expectedProviderOrder); + expect(derived).toEqual(fixture.setup.providers); + + const result = generateTierChains({ schema: 1, providers: derived }, undefined, catalog); + expect(bytes(result.tiers)).toBe(fixture.expectedCanonicalBytes); + expect(result.tiers).toEqual(fixture.expectedTiers); + }); +}); diff --git a/packages/coding-agent/test/autorouting-golden/anthropic-google.json b/packages/coding-agent/test/autorouting-golden/anthropic-google.json new file mode 100644 index 0000000000..7eb4daefc3 --- /dev/null +++ b/packages/coding-agent/test/autorouting-golden/anthropic-google.json @@ -0,0 +1,22 @@ +{ + "catalog": [ + { "provider": "anthropic", "id": "claude-haiku-4-5", "reasoning": true }, + { "provider": "anthropic", "id": "claude-sonnet-5", "reasoning": true }, + { "provider": "anthropic", "id": "claude-sonnet-4-6", "reasoning": true }, + { "provider": "anthropic", "id": "claude-opus-5", "reasoning": true }, + { "provider": "anthropic", "id": "claude-opus-4-8", "reasoning": true }, + { "provider": "google", "id": "gemini-3.5-flash-lite", "reasoning": true }, + { "provider": "google", "id": "gemini-2.5-flash-lite", "reasoning": true }, + { "provider": "google", "id": "gemini-3.5-flash", "reasoning": true }, + { "provider": "google", "id": "gemini-2.5-flash", "reasoning": true }, + { "provider": "google", "id": "gemini-3.1-pro-preview", "reasoning": true }, + { "provider": "google", "id": "gemini-2.5-pro", "reasoning": true } + ], + "setup": { "schema": 1, "providers": ["anthropic", "google"] }, + "expectedTiers": { + "fast": ["anthropic/claude-haiku-4-5", "google/gemini-3.5-flash-lite", "google/gemini-2.5-flash-lite"], + "balanced": ["anthropic/claude-sonnet-5", "anthropic/claude-sonnet-4-6", "google/gemini-3.5-flash", "google/gemini-2.5-flash"], + "strong": ["anthropic/claude-opus-5:high", "anthropic/claude-opus-4-8:high", "google/gemini-3.1-pro-preview", "google/gemini-2.5-pro"] + }, + "expectedCanonicalBytes": "{\"balanced\":[\"anthropic/claude-sonnet-5\",\"anthropic/claude-sonnet-4-6\",\"google/gemini-3.5-flash\",\"google/gemini-2.5-flash\"],\"fast\":[\"anthropic/claude-haiku-4-5\",\"google/gemini-3.5-flash-lite\",\"google/gemini-2.5-flash-lite\"],\"strong\":[\"anthropic/claude-opus-5:high\",\"anthropic/claude-opus-4-8:high\",\"google/gemini-3.1-pro-preview\",\"google/gemini-2.5-pro\"]}" +} diff --git a/packages/coding-agent/test/autorouting-golden/anthropic.json b/packages/coding-agent/test/autorouting-golden/anthropic.json new file mode 100644 index 0000000000..5f78a0437e --- /dev/null +++ b/packages/coding-agent/test/autorouting-golden/anthropic.json @@ -0,0 +1,16 @@ +{ + "catalog": [ + { "provider": "anthropic", "id": "claude-haiku-4-5", "reasoning": true }, + { "provider": "anthropic", "id": "claude-sonnet-5", "reasoning": true }, + { "provider": "anthropic", "id": "claude-sonnet-4-6", "reasoning": true }, + { "provider": "anthropic", "id": "claude-opus-5", "reasoning": true }, + { "provider": "anthropic", "id": "claude-opus-4-8", "reasoning": true } + ], + "setup": { "schema": 1, "providers": ["anthropic"] }, + "expectedTiers": { + "fast": ["anthropic/claude-haiku-4-5"], + "balanced": ["anthropic/claude-sonnet-5", "anthropic/claude-sonnet-4-6"], + "strong": ["anthropic/claude-opus-5:high", "anthropic/claude-opus-4-8:high"] + }, + "expectedCanonicalBytes": "{\"balanced\":[\"anthropic/claude-sonnet-5\",\"anthropic/claude-sonnet-4-6\"],\"fast\":[\"anthropic/claude-haiku-4-5\"],\"strong\":[\"anthropic/claude-opus-5:high\",\"anthropic/claude-opus-4-8:high\"]}" +} diff --git a/packages/coding-agent/test/autorouting-golden/openai.json b/packages/coding-agent/test/autorouting-golden/openai.json new file mode 100644 index 0000000000..03e81a901e --- /dev/null +++ b/packages/coding-agent/test/autorouting-golden/openai.json @@ -0,0 +1,13 @@ +{ + "catalog": [ + { "provider": "openai-codex", "id": "gpt-5.6-terra", "reasoning": true }, + { "provider": "openai-codex", "id": "gpt-5.6-sol", "reasoning": true } + ], + "setup": { "schema": 1, "providers": ["openai-codex"] }, + "expectedTiers": { + "fast": ["openai-codex/gpt-5.6-terra:low"], + "balanced": ["openai-codex/gpt-5.6-terra:medium"], + "strong": ["openai-codex/gpt-5.6-sol:high"] + }, + "expectedCanonicalBytes": "{\"balanced\":[\"openai-codex/gpt-5.6-terra:medium\"],\"fast\":[\"openai-codex/gpt-5.6-terra:low\"],\"strong\":[\"openai-codex/gpt-5.6-sol:high\"]}" +} diff --git a/packages/coding-agent/test/autorouting-golden/policy-derived-provider-order.json b/packages/coding-agent/test/autorouting-golden/policy-derived-provider-order.json new file mode 100644 index 0000000000..8c63af6628 --- /dev/null +++ b/packages/coding-agent/test/autorouting-golden/policy-derived-provider-order.json @@ -0,0 +1,48 @@ +{ + "catalog": [ + { + "provider": "google", + "id": "gemini-3-pro", + "reasoning": true + }, + { + "provider": "anthropic", + "id": "claude-opus-5", + "reasoning": true + }, + { + "provider": "anthropic", + "id": "claude-haiku-5", + "reasoning": false + }, + { + "provider": "openai-codex", + "id": "gpt-5.6-codex", + "reasoning": true + } + ], + "configuredProviderOrder": [ + "openai-codex", + "ghost-provider", + "anthropic" + ], + "expectedProviderOrder": [ + "openai-codex", + "anthropic", + "google" + ], + "setup": { + "schema": 1, + "providers": [ + "openai-codex", + "anthropic", + "google" + ] + }, + "expectedTiers": { + "strong": [ + "anthropic/claude-opus-5:high" + ] + }, + "expectedCanonicalBytes": "{\"strong\":[\"anthropic/claude-opus-5:high\"]}" +} diff --git a/packages/coding-agent/test/autorouting-golden/thin-single-provider.json b/packages/coding-agent/test/autorouting-golden/thin-single-provider.json new file mode 100644 index 0000000000..80b1baf8ef --- /dev/null +++ b/packages/coding-agent/test/autorouting-golden/thin-single-provider.json @@ -0,0 +1,8 @@ +{ + "catalog": [ + { "provider": "qianfan", "id": "deepseek-v3.2", "reasoning": false } + ], + "setup": { "schema": 1, "providers": ["qianfan"] }, + "expectedTiers": {}, + "expectedCanonicalBytes": "{}" +} diff --git a/packages/coding-agent/test/autorouting-private-seam.test.ts b/packages/coding-agent/test/autorouting-private-seam.test.ts new file mode 100644 index 0000000000..d45b6b0bd2 --- /dev/null +++ b/packages/coding-agent/test/autorouting-private-seam.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +/** + * The inactive-autorouting diagnostic is internal. It previously rode on + * `SessionSdkHostOptions`, which `sdk/index.ts` re-exports and `./sdk` publishes, + * so any consumer could inject it. These guard the boundary that replaced it. + */ +const repoRoot = path.resolve(import.meta.dir, "../../.."); +const packageRoot = path.resolve(import.meta.dir, ".."); + +async function readSource(relativePath: string): Promise { + return await Bun.file(path.join(packageRoot, relativePath)).text(); +} + +describe("autorouting private seam boundary", () => { + test("the internal state module is blocked from the published export map", async () => { + const manifest = JSON.parse(await Bun.file(path.join(packageRoot, "package.json")).text()) as { + exports: Record; + }; + // A null entry is what prevents `import "@gajae-code/coding-agent/sdk/host/internal-autorouting-state"`. + expect(manifest.exports["./sdk/host/internal-autorouting-state"]).toBeNull(); + expect(manifest.exports["./sdk/host/internal-autorouting-state.js"]).toBeNull(); + }); + + test("the internal state module is not re-exported from the host barrel", async () => { + const barrel = await readSource("src/sdk/host/index.ts"); + expect(barrel).not.toContain("internal-autorouting-state"); + }); + + test("no published option type accepts the diagnostic flag", async () => { + // Declaration sites only: a consumer must not be able to set this. + for (const relativePath of ["src/sdk/host/host.ts", "src/sdk/host/session-runtime.ts", "src/sdk/bus/index.ts"]) { + const source = await readSource(relativePath); + expect(source).not.toContain("autoroutingInactive?:"); + expect(source).not.toContain("autoroutingInactive:"); + } + }); + + test("the internal module exists and stays inside the package", async () => { + const modulePath = path.join(packageRoot, "src/sdk/host/internal-autorouting-state.ts"); + expect(await fs.exists(modulePath)).toBe(true); + expect(modulePath.startsWith(repoRoot)).toBe(true); + }); +}); diff --git a/packages/coding-agent/test/autorouting-provider-order.test.ts b/packages/coding-agent/test/autorouting-provider-order.test.ts new file mode 100644 index 0000000000..fd46bac7aa --- /dev/null +++ b/packages/coding-agent/test/autorouting-provider-order.test.ts @@ -0,0 +1,210 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "bun:test"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { ModelRegistry } from "@gajae-code/coding-agent/config/model-registry"; +import { + buildProviderSelectionCatalog, + createProviderSelectionPolicy, + type EffectiveProviderAuth, + projectCatalogProviderOrder, + projectProviderOrder, +} from "@gajae-code/coding-agent/config/provider-selection-policy"; +import { resetSettingsForTest, settings } from "@gajae-code/coding-agent/config/settings"; +import { AuthStorage } from "@gajae-code/coding-agent/session/auth-storage"; + +describe("projectProviderOrder", () => { + test("puts the explicit order first and appends catalog order after it", () => { + expect( + projectProviderOrder(["openai-codex", "anthropic"], ["anthropic", "google", "openai-codex", "xai"]), + ).toEqual(["openai-codex", "anthropic", "google", "xai"]); + }); + + test("normalizes and dedupes explicit entries the same way provider selection does", () => { + expect(projectProviderOrder([" Anthropic ", "ANTHROPIC", "", "google"], ["anthropic", "google"])).toEqual([ + "anthropic", + "google", + ]); + }); + + test("keeps explicit providers that are absent from the catalog", () => { + // The projection is order-only; catalog membership is the caller's filter. + expect(projectProviderOrder(["ghost"], ["anthropic"])).toEqual(["ghost", "anthropic"]); + }); + + test("preserves first-wins catalog order and drops blanks", () => { + expect(projectProviderOrder([], ["anthropic", "", "anthropic", "google"])).toEqual(["anthropic", "google"]); + }); + + test("is deterministic across repeated calls", () => { + const first = projectProviderOrder(["b"], ["a", "b", "c"]); + const second = projectProviderOrder(["b"], ["a", "b", "c"]); + expect(first).toEqual(second); + }); +}); + +describe("provider order is auth-independent while ranking is not", () => { + const catalogProviders = ["anthropic", "google", "openai-codex"]; + const catalogModels = ["anthropic/claude", "google/gemini", "openai-codex/gpt"]; + + function policyWith(effectiveAuth: ReadonlyMap) { + return createProviderSelectionPolicy({ + explicitProviderOrder: ["google"], + effectiveAuth, + catalogProviders, + catalogModels, + }); + } + + test("orderedProviders ignores effective auth entirely", () => { + const noAuth = policyWith(new Map()); + const oauthElsewhere = policyWith( + new Map([ + ["anthropic", "key"], + ["openai-codex", "oauth"], + ]), + ); + // Flipping openai-codex into the OAuth band must not reorder the projection. + expect(oauthElsewhere.orderedProviders()).toEqual(noAuth.orderedProviders()); + expect(noAuth.orderedProviders()).toEqual(["google", "anthropic", "openai-codex"]); + }); + + test("rank still bands omitted OAuth providers ahead of the rest", () => { + const policy = policyWith( + new Map([ + ["anthropic", "key"], + ["openai-codex", "oauth"], + ]), + ); + expect(policy.rank("google")).toBe(0); + expect(policy.rank("openai-codex")).toBeLessThan(policy.rank("anthropic")); + }); +}); + +describe("buildProviderSelectionCatalog feeds the projection", () => { + test("catalog spelling is lowercased for comparison keys", () => { + const { catalogProviders } = buildProviderSelectionCatalog([ + { provider: "CustomRouter", id: "m1" }, + { provider: "customrouter", id: "m2" }, + { provider: "anthropic", id: "m3" }, + ] as never); + expect(catalogProviders).toEqual(["customrouter", "anthropic"]); + expect(projectProviderOrder([], catalogProviders)).toEqual(["customrouter", "anthropic"]); + }); +}); + +describe("projectCatalogProviderOrder (the accessor's own implementation)", () => { + // autoroutingProviderOrder() is a one-line call to this function, so these + // exercise the real code path rather than a reimplementation of it. + const catalog = (entries: Array<{ provider: string; id: string }>) => entries as never; + + test("restores the catalog's spelling instead of the normalized key", () => { + // The generator matches provider prefixes case-sensitively, so a lowercased + // id here would silently empty CustomRouter's tiers. + expect(projectCatalogProviderOrder(["customrouter"], catalog([{ provider: "CustomRouter", id: "m1" }]))).toEqual([ + "CustomRouter", + ]); + }); + + test("keeps the first-seen spelling when the catalog disagrees with itself", () => { + expect( + projectCatalogProviderOrder( + [], + catalog([ + { provider: "CustomRouter", id: "a" }, + { provider: "customrouter", id: "b" }, + ]), + ), + ).toEqual(["CustomRouter"]); + }); + + test("drops a declared provider the catalog does not offer", () => { + expect( + projectCatalogProviderOrder(["ghost", "anthropic"], catalog([{ provider: "anthropic", id: "m1" }])), + ).toEqual(["anthropic"]); + }); + + test("puts the declared order ahead of the catalog remainder", () => { + const models = catalog([ + { provider: "anthropic", id: "a" }, + { provider: "google", id: "g" }, + ]); + expect(projectCatalogProviderOrder(["google"], models)).toEqual(["google", "anthropic"]); + }); + + test("returns nothing for an empty catalog", () => { + expect(projectCatalogProviderOrder(["anthropic"], catalog([]))).toEqual([]); + }); +}); + +describe("ModelRegistry.autoroutingProviderOrder (real instance)", () => { + // These call the real accessor, not a reimplementation. The configured-order and + // catalog-drop branches are covered by projectCatalogProviderOrder above and by the + // policy-derived golden fixture; here the accessor's settings read yields no + // explicit order, so the catalog projection is what is exercised. + const cleanups: Array<() => void | Promise> = []; + + beforeEach(() => { + // Do not inherit whatever an earlier test left in the global settings + // singleton: a stray modelProviderOrder would silently reorder the expected + // catalog projection and make these assertions accidental. + resetSettingsForTest(); + }); + + afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); + vi.restoreAllMocks(); + resetSettingsForTest(); + }); + + test("reads no explicit provider order, so the catalog projection is what is asserted", () => { + // Makes the precondition explicit instead of assuming it. + expect(() => settings.getGlobal("modelProviderOrder")).toThrow(); + }); + + async function registry(): Promise { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "gjc-autorouting-order-")); + const authStorage = await AuthStorage.create(path.join(tempDir, "auth.db"), { + configValueResolver: async () => undefined, + }); + cleanups.push(() => authStorage.close()); + cleanups.push(async () => await fs.rm(tempDir, { recursive: true, force: true })); + return new ModelRegistry(authStorage, path.join(tempDir, "models.yml")); + } + + test("declares no parameters, so no caller can bind it to a session", async () => { + const instance = await registry(); + expect(instance.autoroutingProviderOrder.length).toBe(0); + }); + + test("returns only providers the catalog actually offers", async () => { + const instance = await registry(); + const order = instance.autoroutingProviderOrder(); + const catalogProviders = new Set(instance.getAll().map(model => model.provider)); + expect(order.length).toBeGreaterThan(0); + for (const provider of order) expect(catalogProviders.has(provider)).toBe(true); + }); + + test("returns each provider once, in catalog first-wins order", async () => { + const instance = await registry(); + const order = instance.autoroutingProviderOrder(); + expect(new Set(order).size).toBe(order.length); + const catalogOrder = [...new Set(instance.getAll().map(model => model.provider))]; + expect(order).toEqual(catalogOrder); + }); + + test("is unchanged by stored credentials", async () => { + const instance = await registry(); + const before = [...instance.autoroutingProviderOrder()]; + const target = before.at(-1); + expect(target).toBeDefined(); + // The accessor never reads auth, so a new credential cannot reorder anything. + await instance.authStorage.set(target as string, [{ type: "api_key", key: "test-key" }]); + expect([...instance.autoroutingProviderOrder()]).toEqual(before); + }); + + test("is deterministic across repeated calls", async () => { + const instance = await registry(); + expect([...instance.autoroutingProviderOrder()]).toEqual([...instance.autoroutingProviderOrder()]); + }); +}); diff --git a/packages/coding-agent/test/autorouting-settings-contract.test.ts b/packages/coding-agent/test/autorouting-settings-contract.test.ts new file mode 100644 index 0000000000..2f413aa7b5 --- /dev/null +++ b/packages/coding-agent/test/autorouting-settings-contract.test.ts @@ -0,0 +1,344 @@ +import { describe, expect, it, vi } from "bun:test"; +import { createHash } from "node:crypto"; +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { + AUTOROUTING_SELECTOR_MAX_LENGTH, + AUTOROUTING_SELECTOR_PATTERN, + type AutoroutingProvenance, + type AutoroutingSetup, + autoroutingProviderOrderHint, + buildAutoroutingClearPatches, + buildAutoroutingEnabledPatch, + buildAutoroutingSettingsBatch, + evaluateAutoroutingProvenanceState, + isValidAutoroutingSelector, + validateAutoroutingEffective, + validateAutoroutingLocal, + validateAutoroutingProvenance, + validateAutoroutingSetup, +} from "../src/config/autorouting-contract"; +import { canonicalJsonBytes } from "../src/config/autorouting-tier-map"; +import { Settings } from "../src/config/settings"; +import { + type OptionalObjectDef, + reconcileSettingsSchema, + SETTINGS_SCHEMA, + type SettingDef, + type SettingValue, + validateSettingPatch, +} from "../src/config/settings-schema"; + +const fingerprint = (value: unknown): string => createHash("sha256").update(canonicalJsonBytes(value)).digest("hex"); + +const setup: AutoroutingSetup = { + schema: 1, + providers: ["anthropic", "openai-codex"], + models: ["anthropic/claude-opus-5"], +}; +const provenance: AutoroutingProvenance = { + schema: 1, + source: { catalogFingerprint: "a".repeat(64), mapFingerprint: "b".repeat(64), generatorVersion: 1 }, + declarationFingerprint: "c".repeat(64), + tiersFingerprint: fingerprint({ fast: ["anthropic/claude-opus-5"] }), +}; + +function assertNever(value: never): never { + throw new Error(`Unexpected setting definition ${(value as { type: string }).type}`); +} + +function assertSettingDefExhaustive(definition: SettingDef): string { + switch (definition.type) { + case "boolean": + case "string": + case "number": + case "enum": + case "array": + case "record": + case "constrained-record": + case "optional-object": + return definition.type; + default: + return assertNever(definition); + } +} + +type SetupValueIsExact = + SettingValue<"task.autorouting.setup"> extends AutoroutingSetup | undefined + ? AutoroutingSetup | undefined extends SettingValue<"task.autorouting.setup"> + ? true + : false + : false; +const setupValueIsExact: SetupValueIsExact = true; + +void setupValueIsExact; +void assertSettingDefExhaustive; + +const validAutoroutingConfig = { + task: { + autorouting: { + setup, + provenance, + }, + }, +}; + +describe("autorouting typed settings contract", () => { + it("covers optional-object SettingDef union exhaustiveness", () => { + expect(assertSettingDefExhaustive(SETTINGS_SCHEMA["task.autorouting.setup"])).toBe("optional-object"); + }); + + it("infers optional-object SettingValue as the typed object or undefined", () => { + const value: SettingValue<"task.autorouting.setup"> = undefined; + const objectValue: SettingValue<"task.autorouting.setup"> = setup; + expect(value).toBeUndefined(); + expect(objectValue).toEqual(setup); + }); + + it("uses absent optional-object defaults and does not serialize them", () => { + expect(SETTINGS_SCHEMA["task.autorouting.setup"].default).toBeUndefined(); + expect(SETTINGS_SCHEMA["task.autorouting.provenance"].default).toBeUndefined(); + const settings = Settings.isolated(); + expect(settings.get("task.autorouting.setup")).toBeUndefined(); + expect(settings.get("task.autorouting.provenance")).toBeUndefined(); + }); + + it("registers optional-object paths as leaves", () => { + const report = reconcileSettingsSchema({ + task: { autorouting: { setup, provenance } }, + }); + expect(report.report.issues.filter(issue => issue.kind === "unknown")).toEqual([]); + }); + + it("accepts plain objects while rejecting scalar and array optional-object values", () => { + expect(reconcileSettingsSchema(validAutoroutingConfig).report.valid).toBe(true); + expect(reconcileSettingsSchema({ task: { autorouting: { setup: "bad" } } }).report.valid).toBe(false); + expect(reconcileSettingsSchema({ task: { autorouting: { setup: [] } } }).report.valid).toBe(false); + }); + + it("delegates optional-object validation exactly on valid, absent, malformed, and extra-property inputs", () => { + expect(reconcileSettingsSchema({ task: { autorouting: { setup } } }).report.valid).toBe(true); + expect(reconcileSettingsSchema({ task: { autorouting: {} } }).report.valid).toBe(true); + const malformed = reconcileSettingsSchema({ task: { autorouting: { setup: { schema: 1, providers: [] } } } }); + expect(malformed.report.issues.some(issue => issue.path === "task.autorouting.setup.providers")).toBe(true); + const extra = reconcileSettingsSchema({ task: { autorouting: { setup: { ...setup, unexpected: true } } } }); + expect(extra.report.issues.some(issue => issue.path === "task.autorouting.setup.unexpected")).toBe(true); + + const validateSetup = vi.spyOn(SETTINGS_SCHEMA["task.autorouting.setup"], "validate"); + try { + reconcileSettingsSchema({ task: { autorouting: { tiers: { fast: ["a/model"] }, setup } } }); + expect(validateSetup).toHaveBeenCalledTimes(1); + } finally { + validateSetup.mockRestore(); + } + + const noTiers = reconcileSettingsSchema({ + task: { + autorouting: { + setup: { ...setup, providers: [] }, + provenance: { ...provenance, generatedAt: "forbidden" }, + }, + }, + }); + expect(noTiers.report.issues.some(issue => issue.path === "task.autorouting.setup.providers")).toBe(true); + expect(noTiers.report.issues.some(issue => issue.path === "task.autorouting.provenance.generatedAt")).toBe(true); + }); + + it("rejects malformed nested autorouting objects at SDK patch ingress", () => { + expect(validateSettingPatch({ "task.autorouting.setup": setup })).toEqual([]); + expect(validateSettingPatch({ "task.autorouting.setup": { schema: 1, providers: [] } })).toEqual([ + expect.objectContaining({ path: "task.autorouting.setup" }), + ]); + expect( + validateSettingPatch({ "task.autorouting.provenance": { ...provenance, generatedAt: "forbidden" } }), + ).toEqual([expect.objectContaining({ path: "task.autorouting.provenance" })]); + expect(validateSettingPatch({ "task.autorouting.tiers": { fast: ["bare-model"] } })).toEqual([ + expect.objectContaining({ path: "task.autorouting.tiers.fast.0" }), + ]); + }); + it("rejects selectors longer than the routing-evidence bound before execution", () => { + const longId = "m".repeat(AUTOROUTING_SELECTOR_MAX_LENGTH); + expect(isValidAutoroutingSelector(`provider/${longId}`)).toBe(false); + expect(isValidAutoroutingSelector(`provider/${"m".repeat(200)}`)).toBe(true); + // The tiers validator must refuse the same over-long selector at config time. + expect(validateAutoroutingLocal({ tiers: { fast: [`provider/${longId}`] } })).not.toEqual([]); + }); + + it("emits closed nested JSON schemas for setup and provenance", async () => { + const schema = await Bun.file(new URL("../../../schemas/config.schema.json", import.meta.url).pathname).json(); + const autorouting = schema.properties.task.properties.autorouting; + const setupSchema = autorouting.properties.setup; + const provenanceSchema = autorouting.properties.provenance; + expect(setupSchema.additionalProperties).toBe(false); + expect(setupSchema.required).toEqual(["schema", "providers"]); + expect(setupSchema.properties.providers).toMatchObject({ type: "array", minItems: 1, uniqueItems: true }); + expect(setupSchema.properties.models.items.pattern).toBe(AUTOROUTING_SELECTOR_PATTERN); + expect(provenanceSchema.additionalProperties).toBe(false); + expect(provenanceSchema.properties.source.additionalProperties).toBe(false); + expect(provenanceSchema.properties.source.properties.generatorVersion).toMatchObject({ + type: "integer", + minimum: 1, + }); + expect(provenanceSchema.properties.declarationFingerprint.pattern).toBe("^[0-9a-f]{64}$"); + expect(JSON.stringify(provenanceSchema)).not.toContain("generatedAt"); + }); + + it("accepts and rejects the local setup/provenance validator matrix", () => { + expect(validateAutoroutingSetup(setup)).toEqual([]); + expect(validateAutoroutingProvenance(provenance)).toEqual([]); + expect(validateAutoroutingSetup({ schema: 1, providers: ["a", "a"] }).length).toBeGreaterThan(0); + expect(validateAutoroutingSetup({ schema: 1, providers: ["a"], models: ["bare-model"] }).length).toBeGreaterThan( + 0, + ); + expect(validateAutoroutingProvenance({ ...provenance, generatedAt: Date.now() }).length).toBeGreaterThan(0); + expect(validateAutoroutingLocal({ setup, provenance })).toEqual([]); + expect(validateAutoroutingLocal({ enabled: true, setup: { schema: 1, providers: [] } }).length).toBeGreaterThan( + 0, + ); + }); + + it("keeps effective enablement semantics independent of setup and provenance", () => { + expect(validateAutoroutingEffective({ enabled: false, setup, provenance })).toEqual({ active: false }); + expect(validateAutoroutingEffective({ enabled: true, setup, provenance }).active).toBe(false); + expect( + validateAutoroutingEffective({ enabled: true, tiers: { fast: ["anthropic/model"] }, setup, provenance }), + ).toMatchObject({ + active: true, + }); + }); + + it("builds apply/refresh/clear as one three-key batch and toggle as a separate write", () => { + expect(buildAutoroutingSettingsBatch({ tiers: { fast: ["anthropic/model"] }, setup, provenance })).toEqual([ + { path: "task.autorouting.tiers", op: "set", value: { fast: ["anthropic/model"] } }, + { path: "task.autorouting.setup", op: "set", value: setup }, + { path: "task.autorouting.provenance", op: "set", value: provenance }, + ]); + expect(buildAutoroutingClearPatches()).toEqual([ + { path: "task.autorouting.tiers", op: "unset" }, + { path: "task.autorouting.setup", op: "unset" }, + { path: "task.autorouting.provenance", op: "unset" }, + ]); + expect(buildAutoroutingEnabledPatch(true)).toEqual({ path: "task.autorouting.enabled", op: "set", value: true }); + }); + + it("keeps an atomic batch all-or-nothing when a later patch is invalid", async () => { + const settings = Settings.isolated({ + "task.autorouting.tiers": { fast: ["before/model"] }, + "task.autorouting.setup": setup, + "task.autorouting.provenance": provenance, + }); + const before = { + tiers: settings.get("task.autorouting.tiers"), + setup: settings.get("task.autorouting.setup"), + provenance: settings.get("task.autorouting.provenance"), + }; + await expect( + settings.commitAtomicBatch([ + { path: "task.autorouting.tiers", op: "set", value: { fast: ["after/model"] } }, + { path: "task.autorouting.setup", op: "set", value: undefined } as never, + ]), + ).rejects.toThrow(); + expect(settings.get("task.autorouting.tiers")).toEqual(before.tiers); + expect(settings.get("task.autorouting.setup")).toEqual(before.setup); + expect(settings.get("task.autorouting.provenance")).toEqual(before.provenance); + }); + + it("detects stale map, stale catalog, and hand-edited tiers", () => { + const tiers = { fast: ["anthropic/model"] }; + const current = { catalogFingerprint: "a".repeat(64), mapFingerprint: "b".repeat(64), tiers }; + const fresh = { + ...provenance, + source: { + catalogFingerprint: current.catalogFingerprint, + mapFingerprint: current.mapFingerprint, + generatorVersion: 1, + }, + tiersFingerprint: fingerprint(tiers), + }; + expect(evaluateAutoroutingProvenanceState(fresh, current)).toEqual({ + staleMap: false, + staleCatalog: false, + handEdited: false, + }); + expect(evaluateAutoroutingProvenanceState(fresh, { ...current, mapFingerprint: "c".repeat(64) })).toMatchObject({ + staleMap: true, + }); + expect( + evaluateAutoroutingProvenanceState(fresh, { ...current, catalogFingerprint: "d".repeat(64) }), + ).toMatchObject({ + staleCatalog: true, + }); + expect( + evaluateAutoroutingProvenanceState(fresh, { ...current, tiers: { ...tiers, balanced: ["other/model"] } }), + ).toMatchObject({ + handEdited: true, + }); + }); + + it("round-trips an untouched config byte-for-byte with absent optional-object keys", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "autorouting-settings-")); + const agentDir = path.join(root, "agent"); + const cwd = path.join(root, "workspace"); + await fs.mkdir(agentDir, { recursive: true }); + await fs.mkdir(cwd, { recursive: true }); + const config = "configSchemaVersion: 1\ntask:\n autorouting:\n enabled: false\n preset: anthropic\n"; + const configPath = path.join(agentDir, "config.yml"); + await Bun.write(configPath, config); + try { + const settings = await Settings.loadForScope({ cwd, agentDir }); + await settings.flush(); + expect(await Bun.file(configPath).text()).toBe(config); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); +}); + +// Keep the generic type in the test source so tsc verifies the public contract. +const optionalObjectDef: OptionalObjectDef = SETTINGS_SCHEMA["task.autorouting.setup"]; +void optionalObjectDef; + +describe("autoroutingProviderOrderHint", () => { + it("reports no drift when the declaration matches the current priority", () => { + expect(autoroutingProviderOrderHint(["anthropic", "google"], ["anthropic", "google", "xai"])).toEqual({ + reordered: false, + missing: [], + }); + }); + + it("treats an order-preserving subset as unchanged", () => { + expect(autoroutingProviderOrderHint(["anthropic", "xai"], ["anthropic", "google", "xai"])).toEqual({ + reordered: false, + missing: [], + }); + }); + + it("flags a swap of two declared providers", () => { + expect(autoroutingProviderOrderHint(["google", "anthropic"], ["anthropic", "google"])).toEqual({ + reordered: true, + missing: [], + }); + }); + + it("lists declared providers the catalog no longer offers, preserving their spelling", () => { + expect(autoroutingProviderOrderHint(["anthropic", "CustomRouter"], ["anthropic"])).toEqual({ + reordered: false, + missing: ["CustomRouter"], + }); + }); + + it("normalizes case and whitespace the way provider selection does", () => { + expect(autoroutingProviderOrderHint([" Anthropic ", "GOOGLE"], ["anthropic", "google"])).toEqual({ + reordered: false, + missing: [], + }); + }); + + it("ignores duplicate declarations rather than reporting false drift", () => { + expect(autoroutingProviderOrderHint(["anthropic", "anthropic", "google"], ["anthropic", "google"])).toEqual({ + reordered: false, + missing: [], + }); + }); +}); diff --git a/packages/coding-agent/test/autorouting-tier-map-gate.test.ts b/packages/coding-agent/test/autorouting-tier-map-gate.test.ts new file mode 100644 index 0000000000..706f1d7bfe --- /dev/null +++ b/packages/coding-agent/test/autorouting-tier-map-gate.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "bun:test"; +import { checkAutoroutingTierMap, getAutoroutingTierMapGateReport } from "../scripts/check-autorouting-tier-map"; + +type Catalog = Record>>; + +async function committedCatalog(): Promise { + return (await Bun.file(new URL("../../ai/src/models.json", import.meta.url)).json()) as Catalog; +} + +describe("autorouting tier-map CI gate", () => { + it("passes against the committed catalog and landed baseline", async () => { + const result = checkAutoroutingTierMap(await committedCatalog()); + expect(result.ok).toBe(true); + expect(result.report.unlabeledKeys).toEqual([]); + expect(result.report.baselineSkipCount).toBeGreaterThan(0); + }); + + it("reports a synthetic new unlabeled key exactly", async () => { + const catalog = await committedCatalog(); + catalog["new-provider"] = { + "new-model": { + provider: "new-provider", + id: "new-model", + reasoning: false, + input: ["text"], + }, + }; + const result = checkAutoroutingTierMap(catalog); + expect(result.ok).toBe(false); + expect(result.report.unlabeledKeys).toEqual(["new-provider/new-model"]); + }); + + it("accepts an in-scope key when it is explicitly skip-listed", () => { + const catalog = { + qianfan: { + "deepseek-v3.2": { + provider: "qianfan", + id: "deepseek-v3.2", + reasoning: false, + input: ["text"], + }, + }, + }; + const result = getAutoroutingTierMapGateReport(catalog); + expect(result.unlabeledKeys).toEqual([]); + expect(result.skippedKeys).toEqual(["qianfan/deepseek-v3.2"]); + }); + + it("keeps text-capable multimodal models in scope and excludes image-only models", () => { + const catalog = { + vision: { + "text-image": { provider: "vision", id: "text-image", output: ["text", "image"] }, + "image-only": { provider: "vision", id: "image-only", output: ["image"] }, + }, + }; + const result = getAutoroutingTierMapGateReport(catalog); + expect(result.inScopeKeys).toEqual(["vision/text-image"]); + expect(result.unlabeledKeys).toEqual(["vision/text-image"]); + }); + + it("rejects a skip entry with an empty rationale", () => { + const catalog = { + qianfan: { "deepseek-v3.2": { provider: "qianfan", id: "deepseek-v3.2", reasoning: false, input: ["text"] } }, + }; + const result = checkAutoroutingTierMap(catalog, {}, { "qianfan/deepseek-v3.2": { rationale: " " } }); + expect(result.ok).toBe(false); + expect(result.report.invalidSkipKeys).toEqual(["qianfan/deepseek-v3.2"]); + }); + + it("rejects a skip entry whose key violates the selector grammar", () => { + const catalog = { + qianfan: { "deepseek-v3.2": { provider: "qianfan", id: "deepseek-v3.2", reasoning: false, input: ["text"] } }, + }; + const result = checkAutoroutingTierMap(catalog, {}, { "qianfan/glob*": { rationale: "rationale" } }); + expect(result.ok).toBe(false); + expect(result.report.invalidSkipKeys).toEqual(["qianfan/glob*"]); + }); + + it("rejects a skip entry whose key no longer exists in the catalog", () => { + const catalog = { + qianfan: { "deepseek-v3.2": { provider: "qianfan", id: "deepseek-v3.2", reasoning: false, input: ["text"] } }, + }; + const result = checkAutoroutingTierMap(catalog, {}, { "qianfan/removed-key": { rationale: "rationale" } }); + expect(result.ok).toBe(false); + expect(result.report.staleSkipKeys).toEqual(["qianfan/removed-key"]); + }); + + it("rejects a key that is both labeled and skipped", () => { + const catalog = { + qianfan: { "deepseek-v3.2": { provider: "qianfan", id: "deepseek-v3.2", reasoning: false, input: ["text"] } }, + }; + const result = checkAutoroutingTierMap( + catalog, + { "qianfan/deepseek-v3.2": [{ tier: "fast", rank: 1 }] }, + { "qianfan/deepseek-v3.2": { rationale: "rationale" } }, + ); + expect(result.ok).toBe(false); + }); +}); diff --git a/packages/coding-agent/test/autorouting-tier-map.test.ts b/packages/coding-agent/test/autorouting-tier-map.test.ts new file mode 100644 index 0000000000..d319c9490c --- /dev/null +++ b/packages/coding-agent/test/autorouting-tier-map.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "bun:test"; +import type { Model } from "@gajae-code/ai"; +import { AUTOROUTING_SELECTOR_PATTERN, type AutoroutingTier } from "../src/config/autorouting-contract"; +import { + CURATED_TIER_LABELS, + CURATED_TIER_MAP, + type CuratedTierLabels, + computeMapFingerprint, + TIER_MAP_SKIP_LIST, + type TierMapKey, + validateTierMap, +} from "../src/config/autorouting-tier-map"; + +function model(provider: string, id: string, reasoning = true): Model { + return { + provider, + id, + name: id, + api: "openai-completions", + baseUrl: "https://example.invalid", + reasoning, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4096, + }; +} + +function mapWith(labels: CuratedTierLabels, skips: Record = {}) { + return { labels, skips, version: 1 }; +} + +describe("autorouting tier map", () => { + it("keeps every curated key provider-qualified and effort values within the selector union", () => { + const pattern = new RegExp(AUTOROUTING_SELECTOR_PATTERN); + const efforts = new Set(["minimal", "low", "medium", "high", "xhigh"]); + for (const [key, assignments] of Object.entries(CURATED_TIER_LABELS)) { + expect(pattern.test(key)).toBe(true); + for (const assignment of assignments) { + if ("effort" in assignment && assignment.effort !== undefined) + expect(efforts.has(assignment.effort)).toBe(true); + } + } + const typedKey: TierMapKey = "example/provider-model"; + expect(typedKey).toContain("/"); + }); + + it("accepts multi-assignment with one assignment per tier and effort variants", () => { + const labels = { + "example/model": [ + { tier: "fast" as AutoroutingTier, effort: "low" as const, rank: 1 }, + { tier: "balanced" as AutoroutingTier, effort: "medium" as const, rank: 1 }, + { tier: "strong" as AutoroutingTier, effort: "high" as const, rank: 1 }, + ], + } satisfies CuratedTierLabels; + expect(() => validateTierMap(mapWith(labels), [model("example", "model")])).not.toThrow(); + }); + + it("rejects duplicate tier assignments and cross-model provider/tier rank collisions", () => { + const duplicate = { + "example/model": [ + { tier: "fast" as const, rank: 1 }, + { tier: "fast" as const, effort: "low" as const, rank: 2 }, + ], + } satisfies CuratedTierLabels; + expect(() => validateTierMap(mapWith(duplicate), [model("example", "model")])).toThrow("at most one assignment"); + + const collision = { + "example/one": [{ tier: "fast" as const, rank: 1 }], + "example/two": [{ tier: "fast" as const, rank: 1 }], + } satisfies CuratedTierLabels; + expect(() => validateTierMap(mapWith(collision), [model("example", "one"), model("example", "two")])).toThrow( + "collides", + ); + }); + + it("rejects label/skip overlap and effort on non-reasoning models", () => { + const labels = { + "example/model": [{ tier: "fast" as const, effort: "low" as const, rank: 1 }], + } satisfies CuratedTierLabels; + expect(() => + validateTierMap(mapWith(labels, { "example/model": { rationale: "overlap" } }), [ + model("example", "model", false), + ]), + ).toThrow("Effort is only valid"); + expect(() => + validateTierMap(mapWith(labels, { "example/model": { rationale: "overlap" } }), [model("example", "model")]), + ).toThrow("both labeled and skipped"); + }); + + it("moves the fingerprint when labels, skips, or version data changes", () => { + const original = computeMapFingerprint(CURATED_TIER_MAP); + const labels = { + ...CURATED_TIER_LABELS, + "fingerprint/model": [{ tier: "fast" as const, rank: 1 }], + } satisfies CuratedTierLabels; + const changedLabels = computeMapFingerprint({ labels, skips: TIER_MAP_SKIP_LIST, version: 1 }); + expect(changedLabels).not.toBe(original); + const changedSkips = computeMapFingerprint({ + labels: CURATED_TIER_LABELS, + skips: { "fingerprint/model": { rationale: "test" } }, + version: 1, + }); + expect(changedSkips).not.toBe(original); + expect(computeMapFingerprint({ labels: CURATED_TIER_LABELS, skips: TIER_MAP_SKIP_LIST, version: 2 })).not.toBe( + original, + ); + }); +}); diff --git a/packages/coding-agent/test/config-cli.test.ts b/packages/coding-agent/test/config-cli.test.ts index d713927042..c9451c0252 100644 --- a/packages/coding-agent/test/config-cli.test.ts +++ b/packages/coding-agent/test/config-cli.test.ts @@ -6,7 +6,7 @@ import { getConfigRootDir, setAgentDir } from "@gajae-code/utils"; import { YAML } from "bun"; import { inspectConfigFile, runConfigCommand } from "../src/cli/config-cli"; import { FileLockTestHooks } from "../src/config/file-lock"; -import { resetSettingsForTest } from "../src/config/settings"; +import { resetSettingsForTest, settings } from "../src/config/settings"; let testAgentDir = ""; const originalAgentDir = process.env.GJC_CODING_AGENT_DIR; @@ -78,6 +78,12 @@ describe("config CLI schema coverage", () => { expect(parsed.value).toEqual(["claude-opus-4-6", "gpt-5.3-codex"]); }); + it("parses and validates optional autorouting objects as JSON", async () => { + const setup = '{"schema":1,"providers":["anthropic"]}'; + await runConfigCommand({ action: "set", key: "task.autorouting.setup", value: setup, flags: { json: true } }); + expect(settings.get("task.autorouting.setup")).toEqual({ schema: 1, providers: ["anthropic"] }); + }); + it("sets and gets deep-interview ambiguity threshold", async () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); diff --git a/packages/coding-agent/test/model-selector-smart-routing.integration.test.ts b/packages/coding-agent/test/model-selector-smart-routing.integration.test.ts new file mode 100644 index 0000000000..661833380c --- /dev/null +++ b/packages/coding-agent/test/model-selector-smart-routing.integration.test.ts @@ -0,0 +1,486 @@ +import { beforeAll, describe, expect, test, vi } from "bun:test"; +import type { Model } from "@gajae-code/ai"; +import type { AutoroutingSetup, TierMap } from "@gajae-code/coding-agent/config/autorouting-contract"; +import { canonicalJsonBytes } from "@gajae-code/coding-agent/config/autorouting-generator"; +import { Settings } from "@gajae-code/coding-agent/config/settings"; +import type { ModelSelectorComponent } from "@gajae-code/coding-agent/modes/components/model-selector"; +import type { + SmartRoutingPanelComponent, + SmartRoutingPreview, +} from "@gajae-code/coding-agent/modes/components/smart-routing-panel"; +import { + MAX_PANEL_LINE_WIDTH, + SmartRoutingPanelComponent as SmartRoutingPanelClass, +} from "@gajae-code/coding-agent/modes/components/smart-routing-panel"; +import { SelectorController } from "@gajae-code/coding-agent/modes/controllers/selector-controller"; +import { getThemeByName, setThemeInstance } from "@gajae-code/coding-agent/modes/theme/theme"; + +const model = (provider: string, id: string): Model => + ({ provider, id, name: id, api: "openai-responses", contextWindow: 1000, maxTokens: 1000 }) as Model; + +const catalog = [ + model("anthropic", "claude-haiku-4-5"), + model("anthropic", "claude-sonnet-5"), + model("anthropic", "claude-sonnet-4-6"), + model("anthropic", "claude-opus-5"), + model("openai-codex", "gpt-5.6-terra"), + model("openai-codex", "gpt-5.6-sol"), +]; + +const smartProfile = { + name: "smart-test", + displayName: "Smart Test", + requiredProviders: ["anthropic"], + modelMapping: { default: "anthropic/claude-haiku-4-5" }, + source: "user" as const, +}; + +function renderText(component: { render(width: number): string[] }): string { + return component + .render(240) + .join("\n") + .replace(/\x1b\[[0-9;]*m/g, "") + .replace(/\s+/g, " ") + .trim(); +} + +function canonicalBytes(value: unknown): string { + return Buffer.from(canonicalJsonBytes(value)).toString("hex"); +} + +let themeInstance = await getThemeByName("red-claw"); + +function installTheme(): void { + if (!themeInstance) throw new Error("Failed to load test theme"); + setThemeInstance(themeInstance); +} + +function createContext( + options: { + scopedModels?: unknown[]; + settings?: Settings; + noProfiles?: boolean; + providerOrder?: readonly string[]; + } = {}, +) { + const settings = options.settings ?? Settings.isolated(); + const ui = { setFocus: vi.fn(), requestRender: vi.fn(), terminal: { rows: 40, columns: 120 } }; + const editorContainer = { clear: vi.fn(), addChild: vi.fn() }; + const registry = { + getAll: () => catalog, + getAvailable: () => catalog, + refresh: vi.fn(async () => {}), + getError: () => undefined, + getCanonicalModels: () => [], + getCanonicalModelSelections: (query: { candidates?: Model[] } = {}) => + (query.candidates ?? catalog).map(candidate => { + const selector = `${candidate.provider}/${candidate.id}`; + return { + record: { + id: selector, + name: candidate.name, + variants: [{ selector, model: candidate, canonicalId: selector, source: "bundled" }], + }, + model: candidate, + }; + }), + resolveCanonicalModel: () => undefined, + getDiscoverableProviders: () => [], + autoroutingProviderOrder: () => options.providerOrder ?? [...new Set(catalog.map(model => model.provider))], + getModelProfiles: () => (options.noProfiles ? new Map() : new Map([[smartProfile.name, smartProfile]])), + getModelProfile: (name: string) => (name === smartProfile.name ? smartProfile : undefined), + getAvailableModelProfileNames: () => [smartProfile.name], + getApiKeyForProvider: vi.fn(async () => "key"), + getApiKey: vi.fn(async () => "key"), + hasConfiguredProviderAuth: () => false, + }; + const session = { + model: catalog[0], + thinkingLevel: undefined, + sessionId: "smart-routing-test", + scopedModels: options.scopedModels ?? [], + modelRegistry: registry, + getActiveModelProfile: () => undefined, + isFastForProvider: () => false, + isFastForSubagentProvider: () => false, + isFastModeActive: () => false, + }; + const ctx = { + ui, + editorContainer, + editor: {}, + settings, + session, + statusLine: { invalidate: vi.fn() }, + updateEditorBorderColor: vi.fn(), + showStatus: vi.fn(), + showError: vi.fn(), + notifyConfigChanged: vi.fn(async () => {}), + restoreComposer: vi.fn(), + }; + return { ctx, settings, session, editorContainer }; +} + +async function settle(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +async function openPanel(options: Parameters[0] & { smartRoutingOnly?: boolean } = {}): Promise<{ + controller: SelectorController; + selector: ModelSelectorComponent; + panel: SmartRoutingPanelComponent; + settings: Settings; + ctx: ReturnType["ctx"]; +}> { + const { ctx, settings, editorContainer } = createContext(options); + const controller = new SelectorController(ctx as never); + controller.showModelSelector(options.smartRoutingOnly ? { smartRoutingOnly: true } : undefined); + const selector = editorContainer.addChild.mock.calls[0]?.[0] as ModelSelectorComponent; + await settle(); + installTheme(); + if (options.smartRoutingOnly) { + // Nothing to navigate: the standalone entry mounts the panel itself. + } else if ((options.scopedModels?.length ?? 0) > 0) { + selector.__testOpenSmartRoutingPanel(); + } else { + for (let index = 0; index < 20 && selector.__testSelectedPresetRowIdentity() !== "smartRouting"; index++) { + selector.handleInput("\x1b[B"); + } + selector.handleInput("\n"); + } + await settle(); + const panel = selector.__testGetSmartRoutingPanel(); + if (!panel) throw new Error("Smart-routing landing row did not open the panel"); + return { controller, selector, panel, settings, ctx }; +} + +describe("/model smart-routing panel integration", () => { + beforeAll(async () => { + themeInstance = await getThemeByName("red-claw"); + installTheme(); + }); + + test("bare /model reaches the landing row and freezes the preview as the apply payload (AC5/AC6)", async () => { + const { panel, settings, ctx } = await openPanel(); + const preview = panel.getPreviewPayload(); + const observedPatches: unknown[] = []; + const commit = settings.commitAtomicBatchWithCurrent.bind(settings); + vi.spyOn(settings, "commitAtomicBatchWithCurrent").mockImplementation(async builder => { + observedPatches.push(...(await builder({}))); + return commit(builder); + }); + + await panel.__testApply(); + + expect(observedPatches.map(patch => (patch as { path: string }).path)).toEqual([ + "task.autorouting.tiers", + "task.autorouting.setup", + "task.autorouting.provenance", + ]); + expect(canonicalBytes(settings.get("task.autorouting.tiers"))).toBe(canonicalBytes(preview.tiers)); + expect(canonicalBytes(settings.get("task.autorouting.setup"))).toBe(canonicalBytes(preview.setup)); + expect(canonicalBytes(settings.get("task.autorouting.provenance"))).toBe(canonicalBytes(preview.provenance)); + expect( + canonicalBytes({ + tiers: settings.get("task.autorouting.tiers"), + setup: settings.get("task.autorouting.setup"), + provenance: settings.get("task.autorouting.provenance"), + }), + ).toBe(canonicalBytes({ tiers: preview.tiers, setup: preview.setup, provenance: preview.provenance })); + expect(ctx.showStatus).toHaveBeenCalled(); + }); + + test("toggle writes only task.autorouting.enabled (AC8)", async () => { + const { panel, settings } = await openPanel(); + const observedPatches: unknown[] = []; + const commit = settings.commitAtomicBatchWithCurrent.bind(settings); + vi.spyOn(settings, "commitAtomicBatchWithCurrent").mockImplementation(async builder => { + observedPatches.push(...(await builder({}))); + return commit(builder); + }); + + await panel.__testToggle(true); + + expect(observedPatches).toHaveLength(1); + expect(observedPatches[0]).toMatchObject({ path: "task.autorouting.enabled", op: "set", value: true }); + }); + + test("refresh guards hand-edited tiers and proceeds after explicit confirmation (AC7)", async () => { + const { panel, settings } = await openPanel(); + await panel.__testApply(); + settings.override("task.autorouting.tiers", { fast: ["anthropic/hand-edited"] }); + + await panel.__testRefresh(); + expect(panel.mode).toBe("confirming"); + expect(panel.confirmation).toBe("hand-edit"); + await panel.__testConfirm(); + expect(settings.get("task.autorouting.provenance")).toBeDefined(); + expect(panel.mode).toBe("done"); + }); + + test("confirming a hand-edit-guarded Apply commits the edited draft, not a Refresh of the recorded setup (AC6/AC7)", async () => { + const { panel, settings } = await openPanel(); + await panel.__testApply(); + const recordedSetup = settings.get("task.autorouting.setup") as { providers: string[] } | undefined; + expect(recordedSetup?.providers).toBeDefined(); + + // The user edits the draft in-panel, then a hand edit lands in settings underneath them. + const editedProviders = [...(recordedSetup?.providers ?? [])].reverse(); + panel.__testSetProviders(editedProviders); + const editedPreview = panel.getPreviewPayload(); + settings.override("task.autorouting.tiers", { fast: ["anthropic/hand-edited"] }); + + await panel.__testApply(); + expect(panel.mode).toBe("confirming"); + expect(panel.confirmation).toBe("hand-edit"); + + await panel.__testConfirm(); + expect(panel.mode).toBe("done"); + // Regression: confirming previously emitted `refresh`, which discarded the edited draft and + // re-committed the PREVIOUSLY recorded setup. The edited draft must win. + // (`tiers` itself is asserted via the preview payload rather than settings, because the test + // harness injects the hand edit through a higher-precedence override layer.) + expect((settings.get("task.autorouting.setup") as { providers: string[] }).providers).toEqual(editedProviders); + expect(panel.getPreviewPayload().tiers).toEqual(editedPreview.tiers); + }); + + test("clear unsets generated keys while preserving enabled", async () => { + const settings = Settings.isolated({ + "task.autorouting.enabled": true, + }); + const { panel, selector } = await openPanel({ settings }); + await panel.__testApply(); + panel.handleInput("\x1b"); + selector.handleInput("\n"); + await settle(); + const reopened = selector.__testGetSmartRoutingPanel(); + if (!reopened) throw new Error("Smart-routing panel did not reopen"); + reopened.handleInput("c"); + expect(reopened.confirmation).toBe("clear"); + await reopened.__testConfirm(); + expect(settings.get("task.autorouting.tiers")).toEqual({}); + expect(settings.get("task.autorouting.setup")).toBeUndefined(); + expect(settings.get("task.autorouting.provenance")).toBeUndefined(); + expect(settings.get("task.autorouting.enabled")).toBe(true); + }); + + test("stale-provenance indicator renders and scoped sessions are read-only", async () => { + const settings = Settings.isolated({ + "task.autorouting.setup": { schema: 1, providers: ["anthropic"] }, + "task.autorouting.tiers": { fast: ["anthropic/claude-haiku-4-5"] }, + "task.autorouting.provenance": { + schema: 1, + source: { catalogFingerprint: "0".repeat(64), mapFingerprint: "1".repeat(64), generatorVersion: 1 }, + declarationFingerprint: "2".repeat(64), + tiersFingerprint: "3".repeat(64), + }, + }); + const { panel: stalePanel } = await openPanel({ settings }); + expect(renderText(stalePanel)).toContain("Stale generated setup"); + + const scoped = await openPanel({ scopedModels: [{ model: catalog[0] }] }); + expect(renderText(scoped.panel)).toContain("Read-only"); + await scoped.panel.__testToggle(true); + expect(scoped.settings.get("task.autorouting.enabled")).toBe(false); + }); + + test("standalone /routing entry reaches the panel with zero model profiles", async () => { + const { panel, selector } = await openPanel({ smartRoutingOnly: true, noProfiles: true }); + expect(selector.__testViewMode()).toBe("smart-routing"); + expect(renderText(panel)).toContain("Smart routing setup"); + }); + + test("standalone panel cancel closes the selector instead of falling back to the preset landing", async () => { + const { panel, selector, ctx } = await openPanel({ smartRoutingOnly: true, noProfiles: true }); + panel.handleInput("\x1b"); + expect(ctx.restoreComposer).toHaveBeenCalledTimes(1); + expect(selector.__testViewMode()).toBe("smart-routing"); + }); + + test("landing-launched panel cancel still returns to the preset landing", async () => { + const { panel, selector, ctx } = await openPanel(); + panel.handleInput("\x1b"); + expect(ctx.restoreComposer).not.toHaveBeenCalled(); + expect(selector.__testViewMode()).toBe("presets"); + }); +}); + +describe("provider-order derived seeding (Steps 3-4)", () => { + beforeAll(() => { + installTheme(); + }); + + test("seeds the draft from the derived provider priority, not raw catalog iteration", async () => { + const { panel } = await openPanel({ providerOrder: ["openai-codex", "anthropic"] }); + expect(panel.getProviderOrder()).toEqual(["openai-codex", "anthropic"]); + }); + + test("a recorded declaration still wins over the derived seed", async () => { + const settings = Settings.isolated(); + await settings.set("task.autorouting.setup", { schema: 1, providers: ["anthropic"] }); + const { panel } = await openPanel({ settings, providerOrder: ["openai-codex", "anthropic"] }); + expect(panel.getProviderOrder()).toEqual(["anthropic"]); + }); + + test("refuses to open the panel when no providers are available", async () => { + const { ctx, editorContainer } = createContext({ providerOrder: [] }); + const controller = new SelectorController(ctx as never); + controller.showModelSelector(); + const selector = editorContainer.addChild.mock.calls[0]?.[0] as ModelSelectorComponent; + await settle(); + installTheme(); + for (let index = 0; index < 20 && selector.__testSelectedPresetRowIdentity() !== "smartRouting"; index++) { + selector.handleInput("\x1b[B"); + } + selector.handleInput("\n"); + await settle(); + expect(selector.__testGetSmartRoutingPanel()).toBeUndefined(); + expect(selector.__testViewMode()).toBe("presets"); + }); + + test("an external provider-order change updates the hint without discarding an unsaved draft", async () => { + const settings = Settings.isolated(); + const { panel } = await openPanel({ settings, providerOrder: ["anthropic", "openai-codex"] }); + const before = panel.getProviderOrder(); + expect(before.length).toBeGreaterThan(1); + // Reorder in the panel without applying, then let an external settings change land. + panel.handleInput("\x1b[B"); + panel.handleInput("J"); + const edited = panel.getProviderOrder(); + // Guard against a tautology: the edit must actually have changed the draft. + expect(edited).not.toEqual(before); + await settings.set("modelProviderOrder", ["openai-codex"]); + await settle(); + // The unsaved draft must survive the advisory refresh. + expect(panel.getProviderOrder()).toEqual(edited); + }); +}); + +describe("smart-routing panel hostile render boundary", () => { + beforeAll(() => { + installTheme(); + }); + + /** Raw render: keep escapes so the assertions can prove they were stripped. */ + function rawRender(panel: SmartRoutingPanelComponent): string { + return panel.render(120).join("\n"); + } + + const HOSTILE = "\x1b]0;pwned\x07\x1b[2Jbad\x07\tname\nINJECTED-PANEL-ROW\r\nINJECTED-CRLF-ROW"; + + function hostilePanel(): SmartRoutingPanelComponent { + const setup: AutoroutingSetup = { + schema: 1, + providers: [HOSTILE, "anthropic"], + models: [`${HOSTILE}/model`, "x".repeat(400)], + }; + const tiers: TierMap = { + fast: [`${HOSTILE}/fast-model`], + balanced: ["y".repeat(400)], + strong: ["anthropic/claude-opus-5"], + }; + const preview = { + setup, + tiers, + provenance: { + schema: 1 as const, + source: { catalogFingerprint: "c", mapFingerprint: "m", generatorVersion: 1 }, + declarationFingerprint: "d", + tiersFingerprint: "t", + }, + sourceIdentity: { catalogFingerprint: "c", mapFingerprint: "m", generatorVersion: 1 }, + } as unknown as SmartRoutingPreview; + return new SmartRoutingPanelClass({ + setup, + tiers, + enabled: true, + readOnly: false, + stale: false, + preview, + generatePreview: () => preview, + onSelect: () => undefined, + onCancel: () => undefined, + }); + } + + test("strips control sequences from provider, allowlist, preset, and tier rows", () => { + const rendered = rawRender(hostilePanel()); + // Only SGR color codes may survive; OSC/CSI-erase/BEL/tab data must not. + expect(rendered).not.toContain("\x1b]0;"); + expect(rendered).not.toContain("\x1b[2J"); + expect(rendered).not.toContain("\x07"); + expect(rendered).not.toContain("\t"); + expect(rendered.replace(/\x1b\[[0-9;]*m/g, "")).not.toContain("\x1b"); + // The surrounding literal text still renders, so sanitizing did not blank the row. + expect(rendered).toContain("bad"); + expect(rendered).toContain("anthropic"); + }); + + test("keeps every untrusted value on a single row", () => { + // sanitizeText preserves LF and width truncation treats it as zero-width, so an + // embedded newline would otherwise inject rows and evade the one-line cap. + const rendered = rawRender(hostilePanel()); + expect(rendered).not.toContain("INJECTED-PANEL-ROW\n"); + for (const line of rendered.split("\n")) { + const plain = line.replace(/\x1b\[[0-9;]*m/g, ""); + expect(plain.startsWith("INJECTED-PANEL-ROW")).toBe(false); + expect(plain.startsWith("INJECTED-CRLF-ROW")).toBe(false); + } + // Flattening must preserve the text itself on the owning row. + expect(rendered).toContain("INJECTED-PANEL-ROW"); + }); + + test("bounds oversized catalog selectors to the panel width budget", () => { + // Wrapping alone would hide an unbounded value, so measure the longest + // contiguous run of the oversized selector across the whole render. + const plain = rawRender(hostilePanel()) + .replace(/\x1b\[[0-9;]*m/g, "") + .replace(/\s+/g, ""); + const longestRun = Math.max(0, ...(plain.match(/y+/g) ?? []).map(run => run.length)); + expect(longestRun).toBeGreaterThan(0); + expect(longestRun).toBeLessThanOrEqual(MAX_PANEL_LINE_WIDTH); + }); + + test("sanitizes error text raised by a failing preview regeneration", () => { + const baseSetup: AutoroutingSetup = { schema: 1, providers: ["anthropic", "openai-codex"] }; + const preview = { + setup: baseSetup, + tiers: {}, + provenance: { + schema: 1 as const, + source: { catalogFingerprint: "c", mapFingerprint: "m", generatorVersion: 1 }, + declarationFingerprint: "d", + tiersFingerprint: "t", + }, + sourceIdentity: { catalogFingerprint: "c", mapFingerprint: "m", generatorVersion: 1 }, + } as unknown as SmartRoutingPreview; + const panel = new SmartRoutingPanelClass({ + setup: baseSetup, + enabled: false, + readOnly: false, + stale: false, + preview, + generatePreview: () => { + throw new Error(HOSTILE); + }, + onSelect: () => undefined, + onCancel: () => undefined, + }); + // Removing a provider regenerates the preview, so the thrown message reaches #error. + panel.handleInput("x"); + const rendered = rawRender(panel); + expect(rendered.replace(/\x1b\[[0-9;]*m/g, "")).toContain("bad"); + expect(rendered).not.toContain("\x1b]0;"); + expect(rendered).not.toContain("\x1b[2J"); + expect(rendered).not.toContain("\x07"); + // The error string is free-form, so it must also stay on exactly one row. + for (const line of rendered.split("\n")) { + const plain = line.replace(/\x1b\[[0-9;]*m/g, ""); + expect(plain.startsWith("INJECTED-PANEL-ROW")).toBe(false); + expect(plain.startsWith("INJECTED-CRLF-ROW")).toBe(false); + expect(Bun.stringWidth(plain)).toBeLessThanOrEqual(MAX_PANEL_LINE_WIDTH); + } + }); +}); diff --git a/packages/coding-agent/test/notifications-topic-registry.test.ts b/packages/coding-agent/test/notifications-topic-registry.test.ts index 4721053270..acf3471f62 100644 --- a/packages/coding-agent/test/notifications-topic-registry.test.ts +++ b/packages/coding-agent/test/notifications-topic-registry.test.ts @@ -756,7 +756,7 @@ test("preserves a no-provenance endpoint claim before a held create can stage it await creating; expect(reg.endpointAuthority(binding)).toEqual({ state: "unique", sessionId: "B" }); }); -test("publishes exact durable authority generation 171 at serving epoch 87", () => { +test("publishes exact durable authority generation 172 at serving epoch 87", () => { // Generation 58: parser-valid durable-fence promotion and rollback. // Generation 152: a thrown steady heartbeat renewal in the run loop is // contained instead of terminating the daemon (#4200). @@ -787,7 +787,7 @@ test("publishes exact durable authority generation 171 at serving epoch 87", () // monotonic reaction settlement for Telegram notification delivery (#4528). // Generation 169: delivers every ring-positioned session event live through // the bounded, capability-gated directed subscriber leg used by replay. - expect(DAEMON_GENERATION).toBe(171); + expect(DAEMON_GENERATION).toBe(172); expect(SERVING_EPOCH).toBe(87); }); test("archives pending topics into retained inactive records", async () => { diff --git a/packages/coding-agent/test/session-staging-discovery.test.ts b/packages/coding-agent/test/session-staging-discovery.test.ts new file mode 100644 index 0000000000..b6ccc11423 --- /dev/null +++ b/packages/coding-agent/test/session-staging-discovery.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "bun:test"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + listProjectSessionTranscriptFiles, + resolveResumableSession, + SessionManager, +} from "../src/session/session-manager"; +import { isStagedSessionPath, SESSION_STAGING_DIRNAME } from "../src/session/session-staging-paths"; + +async function makeTranscript(filePath: string, cwd: string, id: string): Promise { + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile( + filePath, + `${JSON.stringify({ type: "session", version: 5, id, timestamp: new Date().toISOString(), cwd })}\n`, + ); +} + +describe("staged session discovery exclusion", () => { + it("hides staged project transcripts while retaining a sibling transcript", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "gjc-discovery-cwd-")); + const agentDir = await mkdtemp(path.join(tmpdir(), "gjc-discovery-agent-")); + const scope = path.join(cwd, ".gjc", "sessions", "scope"); + const staged = path.join(scope, SESSION_STAGING_DIRNAME, "attempt.jsonl"); + const sibling = path.join(scope, "sibling.jsonl"); + await makeTranscript(staged, cwd, "staged-id"); + await makeTranscript(sibling, cwd, "sibling-id"); + + const sessions = await SessionManager.listManagedForResumePickerReadOnly(cwd, agentDir); + expect(sessions.map(session => session.path)).toContain(sibling); + expect(sessions.map(session => session.path)).not.toContain(staged); + const resumed = await resolveResumableSession("sibling-id", cwd, undefined, undefined, agentDir); + expect(resumed?.session.path).toBe(sibling); + const stagedResume = await resolveResumableSession("staged-id", cwd, undefined, undefined, agentDir); + expect(stagedResume).toBeUndefined(); + }); + + it("recognizes staging segments independently of session-layer imports", async () => { + const stagedPath = path.join("/tmp", "agent-session", SESSION_STAGING_DIRNAME, "attempt.jsonl"); + expect(isStagedSessionPath(stagedPath)).toBe(true); + expect(isStagedSessionPath(path.join("/tmp", "agent-session", "sibling.jsonl"))).toBe(false); + const source = await readFile(new URL("../src/session/session-staging-paths.ts", import.meta.url), "utf8"); + expect(source).not.toContain("session-manager"); + expect(source).not.toContain("./artifacts"); + }); + + it("excludes sessions//.staging and agent-session/.staging from all four readers individually", async () => { + const cwd = await mkdtemp(path.join(tmpdir(), "gjc-four-reader-cwd-")); + const projectScope = path.join(cwd, ".gjc", "sessions", "cwd-scope"); + const agentSessionScope = path.join(cwd, ".gjc", "agent-session"); + const projectStaged = path.join(projectScope, SESSION_STAGING_DIRNAME, "project-staged.jsonl"); + const agentSessionStaged = path.join(agentSessionScope, SESSION_STAGING_DIRNAME, "agent-staged.jsonl"); + const projectSibling = path.join(projectScope, "project-sibling.jsonl"); + const agentSessionSibling = path.join(agentSessionScope, "agent-sibling.jsonl"); + await makeTranscript(projectStaged, cwd, "project-staged-id"); + await makeTranscript(agentSessionStaged, cwd, "agent-staged-id"); + await makeTranscript(projectSibling, cwd, "project-sibling-id"); + await makeTranscript(agentSessionSibling, cwd, "agent-sibling-id"); + + // Reader 1: the direct project transcript walk must skip both staging subtrees. + const walked = listProjectSessionTranscriptFiles(cwd); + expect(walked).toContain(projectSibling); + expect(walked).toContain(agentSessionSibling); + expect(walked).not.toContain(projectStaged); + expect(walked).not.toContain(agentSessionStaged); + + // Reader 2: explicit resume-picker listing must skip a staged project scope. + const projectPicker = await SessionManager.listForResumePickerReadOnly(cwd, projectScope); + expect(projectPicker.map(session => session.id)).toEqual(["project-sibling-id"]); + const agentSessionPicker = await SessionManager.listForResumePickerReadOnly(cwd, agentSessionScope); + expect(agentSessionPicker.map(session => session.id)).toEqual(["agent-sibling-id"]); + + // Reader 3: managed picker/inventory must reject a managed .staging child. + const managedCwd = await mkdtemp(path.join(tmpdir(), "gjc-four-reader-managed-cwd-")); + const managedAgentDir = await mkdtemp(path.join(tmpdir(), "gjc-four-reader-managed-agent-")); + const managedDestination = SessionManager.managedDestination(managedCwd, managedAgentDir); + const managedParent = SessionManager.create(managedCwd, managedDestination); + await managedParent.flush(); + const managedStaged = path.join(managedDestination.directory, SESSION_STAGING_DIRNAME, "managed-staged.jsonl"); + await makeTranscript(managedStaged, managedCwd, "managed-staged-id"); + const managedPicker = await SessionManager.listManagedForResumePickerReadOnly(managedCwd, managedAgentDir); + expect(managedPicker.map(session => session.id)).not.toContain("managed-staged-id"); + + // Reader 4: global managed inventory and --continue resolution both reject it. + const managedInventory = await SessionManager.listAll(undefined, managedAgentDir); + expect(managedInventory.map(session => session.id)).not.toContain("managed-staged-id"); + const stagedContinue = await resolveResumableSession( + "managed-staged-id", + managedCwd, + undefined, + undefined, + managedAgentDir, + ); + expect(stagedContinue).toBeUndefined(); + const localContinue = await resolveResumableSession("project-staged-id", cwd, projectScope); + expect(localContinue).toBeUndefined(); + }); +}); diff --git a/packages/coding-agent/test/slash-command-builtin-registry.test.ts b/packages/coding-agent/test/slash-command-builtin-registry.test.ts index 0ecc6b0cc3..d34fc40fae 100644 --- a/packages/coding-agent/test/slash-command-builtin-registry.test.ts +++ b/packages/coding-agent/test/slash-command-builtin-registry.test.ts @@ -1,4 +1,8 @@ import { afterEach, beforeAll, describe, expect, it, vi } from "bun:test"; +import type { Model } from "@gajae-code/ai"; +import { generateTierChains } from "@gajae-code/coding-agent/config/autorouting-generator"; +import { CURATED_TIER_MAP } from "@gajae-code/coding-agent/config/autorouting-tier-map"; +import { Settings } from "@gajae-code/coding-agent/config/settings"; import { BUILTIN_SLASH_COMMANDS } from "@gajae-code/coding-agent/extensibility/slash-commands"; import { getCurrentThemeName, initTheme } from "@gajae-code/coding-agent/modes/theme/theme"; import type { InteractiveModeContext } from "@gajae-code/coding-agent/modes/types"; @@ -8,8 +12,24 @@ import { executeBuiltinSlashCommand, lookupBuiltinSlashCommand, } from "@gajae-code/coding-agent/slash-commands/builtin-registry"; +import { buildAutoroutingStatusReport } from "@gajae-code/coding-agent/slash-commands/helpers/autorouting-status"; import { ImageProtocol, TERMINAL } from "@gajae-code/tui"; +const model = (provider: string, id: string): Model => + ({ + provider, + id, + name: id, + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128000, + maxTokens: 4096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + headers: {}, + compat: {}, + }) as unknown as Model; + const mutableTerminal = TERMINAL as unknown as { imageProtocol: ImageProtocol | null }; const originalImageProtocol = mutableTerminal.imageProtocol; @@ -446,3 +466,152 @@ describe("builtin /theme slash command", () => { expect(String(showError.mock.calls[0]?.[0])).toContain('Unknown theme "not-a-theme"'); }); }); + +describe("builtin /routing slash command", () => { + function createRoutingTuiRuntime(settingsValues: Record = {}) { + const showModelSelector = vi.fn(); + const showStatus = vi.fn(); + const showError = vi.fn(); + const setText = vi.fn(); + const settings = Settings.isolated(settingsValues as never); + // Mirrors SelectorController.setAutoroutingEnabled: the only writer the + // command is allowed to use, so scoped/read-only guards stay in one place. + const setAutoroutingEnabled = vi.fn(async (enabled: boolean) => { + settings.set("task.autorouting.enabled", enabled); + }); + const chatContainer = { addChild: vi.fn() }; + const ctx = { + showModelSelector, + showStatus, + showError, + settings, + setAutoroutingEnabled, + chatContainer, + ui: { requestRender: vi.fn() }, + editor: { setText }, + } as unknown as InteractiveModeContext; + + return { + runtime: { ctx, handleBackgroundCommand: () => undefined }, + showModelSelector, + setAutoroutingEnabled, + settings, + setText, + }; + } + + it("opens the smart-routing panel directly when invoked without arguments", async () => { + const { runtime, showModelSelector, setText } = createRoutingTuiRuntime(); + + expect(await executeBuiltinSlashCommand("/routing", runtime)).toBe(true); + + expect(showModelSelector).toHaveBeenCalledWith({ smartRoutingOnly: true }); + expect(setText).toHaveBeenCalledWith(""); + }); + + it("routes the toggle through the guarded controller entry point", async () => { + const { runtime, showModelSelector, setAutoroutingEnabled, settings } = createRoutingTuiRuntime(); + + expect(await executeBuiltinSlashCommand("/routing on", runtime)).toBe(true); + expect(setAutoroutingEnabled).toHaveBeenLastCalledWith(true); + expect(settings.get("task.autorouting.enabled")).toBe(true); + + expect(await executeBuiltinSlashCommand("/routing off", runtime)).toBe(true); + expect(setAutoroutingEnabled).toHaveBeenLastCalledWith(false); + expect(settings.get("task.autorouting.enabled")).toBe(false); + expect(showModelSelector).not.toHaveBeenCalled(); + }); + + it("reports settings-derived status labels", () => { + const base = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: ["anthropic/model"] }, + }); + const snapshot = () => ({ + effective: base.getEffectiveAutorouting(), + tiers: base.get("task.autorouting.tiers"), + provenance: base.get("task.autorouting.provenance"), + }); + expect(buildAutoroutingStatusReport(snapshot())).toContain("Autorouting: on (hand-authored tiers)"); + expect(buildAutoroutingStatusReport({ ...snapshot(), provenance: undefined })).toContain("hand-authored tiers"); + const catalog = [ + model("anthropic", "claude-haiku-4-5"), + model("anthropic", "claude-sonnet-5"), + model("anthropic", "claude-opus-5"), + ]; + const generated = generateTierChains({ schema: 1, providers: ["anthropic"] }, CURATED_TIER_MAP, catalog); + const generatedSettings = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": generated.tiers, + "task.autorouting.provenance": { + schema: 1, + source: generated.sourceIdentity, + declarationFingerprint: generated.declarationFingerprint, + tiersFingerprint: generated.tiersFingerprint, + }, + }); + const generatedSnapshot = { + effective: generatedSettings.getEffectiveAutorouting(), + tiers: generatedSettings.get("task.autorouting.tiers"), + provenance: generatedSettings.get("task.autorouting.provenance"), + }; + const generatedReport = buildAutoroutingStatusReport(generatedSnapshot); + expect(generatedReport).toContain("Autorouting: on (generated)"); + expect( + buildAutoroutingStatusReport({ + ...generatedSnapshot, + provenance: { ...generatedSnapshot.provenance!, tiersFingerprint: "0".repeat(64) }, + }), + ).toContain("generated, hand-edited"); + expect( + buildAutoroutingStatusReport({ + ...generatedSnapshot, + provenance: { ...generatedSnapshot.provenance!, tiersFingerprint: "bad" }, + }), + ).toContain("hand-authored tiers"); + expect( + buildAutoroutingStatusReport({ + effective: Settings.isolated().getEffectiveAutorouting(), + tiers: undefined, + provenance: undefined, + }), + ).toContain("Autorouting: off"); + }); + + it("strips terminal control sequences from hand-edited selectors", () => { + // The selector grammar rejects control bytes before they can reach status + // rendering, so malformed hand-edited tiers fail closed. + const settings = Settings.isolated({ "task.autorouting.enabled": true } as never); + settings.set("task.autorouting.tiers", { + balanced: ["anthropic/\x1b]0;pwned\x07evil-model"], + } as never); + + const report = buildAutoroutingStatusReport({ + effective: settings.getEffectiveAutorouting(), + tiers: settings.get("task.autorouting.tiers"), + provenance: settings.get("task.autorouting.provenance"), + }); + + expect(report).toContain("Autorouting: off"); + expect(report).not.toContain("\x1b"); + expect(report).not.toContain("\x07"); + }); + + it("bounds a pathologically long chain", () => { + const settings = Settings.isolated({ "task.autorouting.enabled": true } as never); + settings.set("task.autorouting.tiers", { + fast: Array.from({ length: 40 }, (_, index) => `anthropic/model-${index}`), + } as never); + + const fastLine = buildAutoroutingStatusReport({ + effective: settings.getEffectiveAutorouting(), + tiers: settings.get("task.autorouting.tiers"), + provenance: settings.get("task.autorouting.provenance"), + }) + .split("\n") + .find(line => line.trimStart().startsWith("fast:")); + + expect(fastLine).toBeDefined(); + expect(fastLine?.length).toBeLessThanOrEqual(220); + }); +}); diff --git a/packages/coding-agent/test/task-autorouting-preflight.test.ts b/packages/coding-agent/test/task-autorouting-preflight.test.ts new file mode 100644 index 0000000000..30438ea70d --- /dev/null +++ b/packages/coding-agent/test/task-autorouting-preflight.test.ts @@ -0,0 +1,1172 @@ +import { describe, expect, it, spyOn } from "bun:test"; +import type { Dirent } from "node:fs"; +import { mkdir, mkdtemp, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { FallbackTriggerClass } from "@gajae-code/ai/utils/fallback-transport"; +import { getTerminalId } from "@gajae-code/tui"; +import { getTerminalSessionsDir } from "@gajae-code/utils"; +import { AsyncJobManager } from "../src/async"; +import { Settings } from "../src/config/settings"; +import * as sdkModule from "../src/sdk"; +import { ArtifactManager } from "../src/session/artifacts"; +import { ManagedTreeMoveOutcomeError } from "../src/session/internal/managed-session-storage"; +import { resolveResumableSession, SessionManager } from "../src/session/session-manager"; +import { + type AutoroutingPreflightFailure, + classifyAutoroutingPreflightFailure, + runSubprocess, + runSubprocessOnce, +} from "../src/task/executor"; +import { + type AutoroutingAttempt, + type AutoroutingAttemptCode, + assertRoutingEvidenceInvariant, + type SubagentLifecyclePayload, + TASK_SUBAGENT_LIFECYCLE_CHANNEL, + type TaskRoutingEvidence, +} from "../src/task/types"; +import { EventBus } from "../src/utils/event-bus"; + +const agent = { + name: "task", + description: "test agent", + systemPrompt: "test", + source: "bundled" as const, +}; + +const routing: TaskRoutingEvidence = { + tier: "balanced", + requestedSelector: "anthropic/model", + effectiveModel: "anthropic/model", + substitutions: [], +}; + +type ProbePhase = { kind: "pass" } | { kind: "failure"; failure: AutoroutingPreflightFailure }; + +type DurablePhase = + | { kind: "accepted" } + | { kind: "failure"; failure: AutoroutingPreflightFailure } + | { kind: "prepare_throw"; failure: AutoroutingPreflightFailure } + | { kind: "post_fence"; class: FallbackTriggerClass } + | { kind: "rename_failure" } + | { kind: "uncertain_publish" } + | { kind: "reservation_failure" }; + +type CandidateScript = { + selector: string; + probe: ProbePhase; + durable: DurablePhase; +}; + +type ResidueSnapshot = { + finalBytes: string | null; + breadcrumbBytes: string | null; + listing: string[]; + artifactTree: string; + allocatedIds: readonly string[]; + agentUris: string[]; + resumeVisible: boolean; +}; + +type LedgerRun = { + attempts: AutoroutingAttempt[]; + failedSnapshots: Array<{ before: ResidueSnapshot; after: ResidueSnapshot }>; + finalPath: string; + artifactRoot: string; + stagingRoot: string; + finalExists: boolean; + finalText: string | null; + artifactTree: string; + stagingTree: string; + listing: string[]; + parentListing: string[]; + allocatedIds: readonly string[]; + agentUris: string[]; + sessionInitCount: number; + liveHandles: number; + lifecycleStarts: number; + modelFallbackSwitched: number; + parentModelSubstitution: boolean; + resumeVisible: boolean; + terminal: "preflight_exhausted" | "post_acceptance_failure" | "accepted" | undefined; + managed: boolean; + uncertainArtifactId: string | undefined; +}; + +type HarnessContext = { + root: string; + cwd: string; + agentDir: string; + finalPath: string; + artifactRoot: string; + stagingRoot: string; + parentArtifacts: ArtifactManager; + parentManager?: SessionManager; + managed: boolean; +}; + +const fallbackClasses: FallbackTriggerClass[] = ["rate_limit", "quota", "auth", "server", "unknown", "other"]; + +function retryCode(failure: AutoroutingPreflightFailure): AutoroutingAttemptCode { + if (failure.kind === "local" && failure.op === "auth_resolve") return "credential_unavailable"; + if ( + failure.kind === "local" && + (failure.op === "session_open" || failure.op === "tool_bootstrap") && + failure.transient + ) + return "spawn_transient_retry"; + if (failure.kind === "local" && (failure.op === "preflight_validation" || !failure.transient)) + return "config_invalid_terminal"; + return "unclassified_terminal"; +} + +async function snapshotTree(root: string, skipStaging = false): Promise { + const entries: string[] = []; + const walk = async (directory: string, relative: string): Promise => { + let children: Dirent[]; + try { + children = await readdir(directory, { withFileTypes: true }); + } catch { + return; + } + for (const child of children.sort((left, right) => left.name.localeCompare(right.name))) { + if (/^\.gjc-(?:exact-(?:replace-destination|unlink-placeholder)|receipt-remove)-/.test(child.name)) continue; + if (skipStaging && child.isDirectory() && child.name === ".staging") continue; + const childPath = path.join(directory, child.name); + const childRelative = path.join(relative, child.name); + if (child.isDirectory()) { + await walk(childPath, childRelative); + continue; + } + if (child.isFile()) entries.push(`${childRelative}\0${(await readFile(childPath)).toString("base64")}`); + } + }; + await walk(root, ""); + return JSON.stringify(entries.sort()); +} + +async function bytesOrNull(filePath: string): Promise { + try { + return (await readFile(filePath)).toString("base64"); + } catch { + return null; + } +} + +async function breadcrumbBytes(): Promise { + const terminalId = getTerminalId(); + if (!terminalId) return null; + return bytesOrNull(path.join(getTerminalSessionsDir(), terminalId)); +} + +async function enumerateAgentUris(root: string): Promise { + let names: string[] = []; + try { + names = await readdir(root); + } catch { + return []; + } + return names + .filter(name => /^(\d+)\.[^.]+\.log$/u.test(name)) + .map(name => `agent://${name.slice(0, name.indexOf("."))}`) + .sort(); +} + +async function createHarnessContext(managed: boolean): Promise { + const root = await mkdtemp(path.join(tmpdir(), managed ? "gjc-preflight-managed-" : "gjc-preflight-")); + const cwd = path.join(root, "cwd"); + const agentDir = path.join(root, "agent"); + await mkdir(cwd, { recursive: true }); + await mkdir(agentDir, { recursive: true }); + if (managed) { + const parentManager = SessionManager.create(cwd, SessionManager.managedDestination(cwd, agentDir)); + await parentManager.flush(); + const parentArtifacts = parentManager.getArtifactManager(); + if (!parentArtifacts) throw new Error("managed parent artifact manager unavailable"); + await parentArtifacts.save("parent-sibling", "tool"); + const finalPath = path.join(parentArtifacts.dir, "candidate.jsonl"); + return { + root, + cwd, + agentDir, + finalPath, + artifactRoot: parentArtifacts.dir, + stagingRoot: path.join(parentArtifacts.dir, ".staging"), + parentArtifacts, + parentManager, + managed, + }; + } + const finalPath = path.join(root, "candidate.jsonl"); + const parentArtifacts = new ArtifactManager(path.join(root, "candidate")); + await parentArtifacts.save("parent-sibling", "tool"); + return { + root, + cwd, + agentDir, + finalPath, + artifactRoot: parentArtifacts.dir, + stagingRoot: path.join(root, ".staging"), + parentArtifacts, + managed, + }; +} + +async function listContext(ctx: HarnessContext): Promise { + const sessions = ctx.managed + ? await SessionManager.listManagedForResumePickerReadOnly(ctx.cwd, ctx.agentDir) + : await SessionManager.listForResumePickerReadOnly(ctx.cwd, path.dirname(ctx.finalPath)); + return sessions.map(session => `${session.id}:${session.path}`).sort(); +} + +async function residueSnapshot(ctx: HarnessContext): Promise { + const sessionArg = path.basename(ctx.finalPath, ".jsonl"); + const resume = ctx.managed + ? await resolveResumableSession(sessionArg, ctx.cwd, undefined, undefined, ctx.agentDir) + : await resolveResumableSession(sessionArg, ctx.cwd, path.dirname(ctx.finalPath)); + return { + finalBytes: await bytesOrNull(ctx.finalPath), + breadcrumbBytes: await breadcrumbBytes(), + listing: await listContext(ctx), + artifactTree: await snapshotTree(ctx.artifactRoot), + allocatedIds: ctx.parentArtifacts.getAllocatedIds(), + agentUris: await enumerateAgentUris(ctx.artifactRoot), + resumeVisible: resume !== undefined, + }; +} + +async function openStagedCandidate(ctx: HarnessContext, attemptId: string): Promise { + if (!ctx.managed) return SessionManager.openStaged(ctx.finalPath, undefined, attemptId); + const store = ctx.parentArtifacts.getManagedStore(); + if (!store) throw new Error("managed artifact store unavailable"); + const destination = SessionManager.nestedManagedDestination(store, ctx.parentArtifacts.dir); + return SessionManager.openStagedNestedManaged(ctx.finalPath, destination, store, undefined, attemptId); +} + +/** + * Scripted lifecycle harness: provider transport and AgentSession callbacks are not injectable + * through the public executor seam, so phases are typed here while every session/artifact, + * publication, discard, listing, and rollback assertion uses the real runtime APIs. + */ +async function runScriptedLedger(scripts: CandidateScript[], managed = false): Promise { + const ctx = await createHarnessContext(managed); + let uncertainArtifactId: string | undefined; + const attempts: AutoroutingAttempt[] = []; + const failedSnapshots: Array<{ before: ResidueSnapshot; after: ResidueSnapshot }> = []; + const consumed = new Set(); + let terminal: LedgerRun["terminal"]; + let liveHandles = 0; + let lifecycleStarts = 0; + let modelFallbackSwitched = 0; + const parentModelSubstitution = false; + for (const script of scripts) { + if (consumed.size >= 3 || consumed.has(script.selector)) continue; + consumed.add(script.selector); + if (script.probe.kind === "failure") { + const code = retryCode(script.probe.failure); + attempts.push({ selector: script.selector, phase: "probe", code }); + if (code !== "spawn_transient_retry" && code !== "credential_unavailable") { + terminal = "preflight_exhausted"; + break; + } + continue; + } + attempts.push({ selector: script.selector, phase: "probe", code: "probe_passed" }); + if (script.durable.kind === "rename_failure") await writeFile(ctx.finalPath, "pre-existing-final-bytes"); + const before = await residueSnapshot(ctx); + let manager: SessionManager | undefined; + try { + if (script.durable.kind === "prepare_throw") throw script.durable.failure; + manager = await openStagedCandidate(ctx, `attempt-${consumed.size}`); + } catch (error) { + await manager?.discardStaged(); + await manager?.discardStaged(); + const failure = + script.durable.kind === "prepare_throw" + ? script.durable.failure + : classifyAutoroutingPreflightFailure(error, "session_open"); + const code = retryCode(failure); + attempts.push({ selector: script.selector, phase: "durable", code }); + const after = await residueSnapshot(ctx); + failedSnapshots.push({ before, after }); + if (code !== "spawn_transient_retry" && code !== "credential_unavailable") { + terminal = "preflight_exhausted"; + break; + } + continue; + } + const durable = script.durable; + let stagedId: string | undefined; + if (!ctx.managed) { + stagedId = await manager.saveArtifact(`candidate-${script.selector}`, "tool"); + if (stagedId !== undefined) + manager.appendMessage({ role: "user", content: `artifact://${stagedId}`, timestamp: Date.now() }); + } + if (ctx.managed && durable.kind === "rename_failure") { + await manager.flush(); + await manager.discardStaged(); + await manager.discardStaged(); + attempts.push({ selector: script.selector, phase: "durable", code: "post_acceptance_failure" }); + const after = await residueSnapshot(ctx); + failedSnapshots.push({ before, after }); + terminal = "post_acceptance_failure"; + break; + } + if (ctx.managed && durable.kind === "uncertain_publish") { + manager.appendSessionInit({ systemPrompt: "test", task: script.selector, tools: [] }); + const uncertainArtifact = await manager.saveArtifact(`candidate-${script.selector}`, "tool"); + await manager.flush(); + const store = ctx.parentArtifacts.getManagedStore(); + if (!store) throw new Error("managed artifact store unavailable"); + const realMove = store.moveFileNoReplace.bind(store); + // Native completes the rename and still fails to prove durability/identity. + const move = spyOn(store, "moveFileNoReplace").mockImplementation((source, destination, expected, options) => { + realMove(source, destination, expected, options); + throw new ManagedTreeMoveOutcomeError("managed_publish_fsync_failed", false); + }); + try { + await expect(manager.commitStagedNestedManaged()).rejects.toThrow(/committed without proof/); + } finally { + move.mockRestore(); + } + // Mirror the executor's full compensation: post-fence rollback, then the + // pre-fence discard its `finally` always runs when the fence never opened. + await manager.rollbackCommittedStaged(); + await manager.discardStaged(); + await manager.discardStaged(); + uncertainArtifactId = uncertainArtifact; + attempts.push({ selector: script.selector, phase: "durable", code: "post_acceptance_failure" }); + const after = await residueSnapshot(ctx); + failedSnapshots.push({ before, after }); + terminal = "post_acceptance_failure"; + break; + } + if (ctx.managed && durable.kind === "accepted") { + manager.appendSessionInit({ systemPrompt: "test", task: script.selector, tools: [] }); + await manager.commitStagedNestedManaged(); + await manager.flush(); + liveHandles++; + lifecycleStarts++; + attempts.push({ selector: script.selector, phase: "durable", code: "accepted" }); + terminal = "accepted"; + break; + } + if (durable.kind === "failure") { + await manager.flush(); + await manager.discardStaged(); + await manager.discardStaged(); + const code = retryCode(durable.failure); + attempts.push({ selector: script.selector, phase: "durable", code }); + const after = await residueSnapshot(ctx); + failedSnapshots.push({ before, after }); + if (code !== "spawn_transient_retry" && code !== "credential_unavailable") { + terminal = "preflight_exhausted"; + break; + } + continue; + } + if (durable.kind === "rename_failure") { + await expect(manager.commitStaged()).rejects.toThrow(); + await manager.discardStaged(); + attempts.push({ selector: script.selector, phase: "durable", code: "post_acceptance_failure" }); + const after = await residueSnapshot(ctx); + failedSnapshots.push({ before, after }); + terminal = "post_acceptance_failure"; + break; + } + if (durable.kind === "reservation_failure") { + const stagedArtifacts = manager.getArtifactManager(); + if (!stagedArtifacts) throw new Error("staged artifact manager unavailable"); + (stagedArtifacts as unknown as { listFiles: () => Promise }).listFiles = async () => { + throw new Error("injected reservation failure"); + }; + await expect(manager.commitStaged()).rejects.toThrow("injected reservation failure"); + await manager.discardStaged(); + attempts.push({ selector: script.selector, phase: "durable", code: "post_acceptance_failure" }); + terminal = "post_acceptance_failure"; + break; + } + if (durable.kind === "post_fence") { + await manager.commitStaged(); + manager.appendSessionInit({ systemPrompt: "test", task: script.selector, tools: [] }); + await manager.flush(); + modelFallbackSwitched += 0; + attempts.push({ selector: script.selector, phase: "durable", code: "post_acceptance_failure" }); + terminal = "post_acceptance_failure"; + break; + } + await manager.commitStaged(); + manager.appendSessionInit({ systemPrompt: "test", task: script.selector, tools: [] }); + await manager.flush(); + liveHandles++; + lifecycleStarts++; + attempts.push({ selector: script.selector, phase: "durable", code: "accepted" }); + terminal = "accepted"; + break; + } + if (!terminal && consumed.size >= 3) terminal = "preflight_exhausted"; + const finalText = await bytesOrNull(ctx.finalPath); + const decodedFinalText = finalText === null ? null : Buffer.from(finalText, "base64").toString("utf8"); + const sessionInitCount = decodedFinalText?.match(/"type":"session_init"/gu)?.length ?? 0; + const parentListing = await listContext(ctx); + const finalResume = ctx.managed + ? await resolveResumableSession( + path.basename(ctx.finalPath, ".jsonl"), + ctx.cwd, + undefined, + undefined, + ctx.agentDir, + ) + : await resolveResumableSession(path.basename(ctx.finalPath, ".jsonl"), ctx.cwd, path.dirname(ctx.finalPath)); + return { + attempts, + failedSnapshots, + finalPath: ctx.finalPath, + artifactRoot: ctx.artifactRoot, + stagingRoot: ctx.stagingRoot, + finalExists: finalText !== null, + finalText: decodedFinalText, + artifactTree: await snapshotTree(ctx.artifactRoot), + stagingTree: await snapshotTree(ctx.stagingRoot), + listing: parentListing, + parentListing, + allocatedIds: ctx.parentArtifacts.getAllocatedIds(), + agentUris: await enumerateAgentUris(ctx.artifactRoot), + resumeVisible: finalResume !== undefined, + sessionInitCount, + liveHandles, + lifecycleStarts, + modelFallbackSwitched, + parentModelSubstitution, + terminal, + managed, + uncertainArtifactId, + }; +} + +function transientSessionFailure(): AutoroutingPreflightFailure { + return { kind: "local", op: "session_open", transient: true }; +} + +function invalidConfigFailure(): AutoroutingPreflightFailure { + return { kind: "local", op: "preflight_validation", transient: false }; +} + +describe("autorouting preflight contract", () => { + it("publishes no durable candidate through the real executor/event bus before the acceptance fence", async () => { + const root = await mkdtemp(path.join(tmpdir(), "gjc-real-preflight-fence-")); + const finalPath = path.join(root, "candidate.jsonl"); + const eventBus = new EventBus(); + const lifecycle: SubagentLifecyclePayload[] = []; + eventBus.on(TASK_SUBAGENT_LIFECYCLE_CHANNEL, payload => { + lifecycle.push(payload as SubagentLifecyclePayload); + }); + const jobs = new AsyncJobManager({ maxRunningJobs: 2, onJobComplete: async () => {} }); + AsyncJobManager.setInstance(jobs); + const model = { + provider: "test", + id: "model", + name: "model", + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128_000, + maxTokens: 4_096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + headers: {}, + compat: {}, + } as never; + const result = await runSubprocessOnce({ + cwd: root, + agent, + task: "real preflight", + assignment: "real preflight", + index: 0, + id: "real-preflight", + modelOverride: ["test/model"], + settings: Settings.isolated(), + modelRegistry: { + authStorage: {}, + getAvailable: () => [model], + getApiKey: async () => "key", + } as never, + preflightDurable: true, + autoroutingAttemptId: "../escaped", + sessionFile: finalPath, + eventBus, + }); + expect(result.preflightFenceCrossed).toBe(false); + expect(result.preflightFailure).toEqual({ kind: "local", op: "session_open", transient: false }); + expect(jobs.getLiveHandle("real-preflight")).toBeUndefined(); + expect(lifecycle).toEqual([]); + await expect(stat(finalPath)).rejects.toThrow(); + }); + + it("fails closed to terminal when the pre-fence discard cleanup itself fails", async () => { + const root = await mkdtemp(path.join(tmpdir(), "gjc-real-preflight-discard-")); + const finalPath = path.join(root, "candidate.jsonl"); + const jobs = new AsyncJobManager({ maxRunningJobs: 2, onJobComplete: async () => {} }); + AsyncJobManager.setInstance(jobs); + const model = { + provider: "test", + id: "model", + name: "model", + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128_000, + maxTokens: 4_096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + headers: {}, + compat: {}, + } as never; + // A transient tool-bootstrap failure would normally ADVANCE to the next candidate. When the + // pre-fence discard cleanup also fails, candidate-owned staging residue may survive, so the + // zero-residue guarantee no longer holds and the attempt must fail closed instead. + const bootstrapSpy = spyOn(sdkModule, "createAgentSession").mockRejectedValue( + Object.assign(new Error("transient bootstrap"), { transient: true }), + ); + const discardSpy = spyOn(SessionManager.prototype, "discardStaged").mockRejectedValue( + new Error("discard-cleanup-failed"), + ); + try { + const result = await runSubprocessOnce({ + cwd: root, + agent, + task: "discard cleanup failure", + assignment: "discard cleanup failure", + index: 0, + id: "discard-cleanup", + modelOverride: ["test/model"], + settings: Settings.isolated(), + modelRegistry: { + authStorage: {}, + getAvailable: () => [model], + getApiKey: async () => "key", + } as never, + preflightDurable: true, + autoroutingAttemptId: "discard-cleanup", + sessionFile: finalPath, + }); + expect(result.preflightFenceCrossed).toBe(false); + expect(result.preflightFailure).toEqual({ kind: "local", op: "preflight_validation", transient: false }); + expect(result.error ?? "").toContain("Cleanup failure"); + } finally { + discardSpy.mockRestore(); + bootstrapSpy.mockRestore(); + } + }); + + it("stops the public runSubprocess ledger and preserves the cleanup diagnostic on failed pre-fence discard", async () => { + const root = await mkdtemp(path.join(tmpdir(), "gjc-real-preflight-ledger-")); + const jobs = new AsyncJobManager({ maxRunningJobs: 2, onJobComplete: async () => {} }); + AsyncJobManager.setInstance(jobs); + const model = { + provider: "test", + id: "model", + name: "model", + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128_000, + maxTokens: 4_096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + headers: {}, + compat: {}, + } as never; + // A NON-transient bootstrap failure must terminalize the ledger immediately: the recorded + // code is config_invalid_terminal and no further candidate may be attempted. Before the fix, + // any returned attempt code was treated as "advance", so terminal codes still advanced. + const bootstrapSpy = spyOn(sdkModule, "createAgentSession").mockRejectedValue( + new Error("non-transient bootstrap failure"), + ); + try { + const result = await runSubprocess({ + cwd: root, + agent, + task: "ledger stop", + assignment: "ledger stop", + index: 0, + id: "ledger-stop", + runMode: "initial", + settings: Settings.isolated(), + modelRegistry: { + authStorage: {}, + getAvailable: () => [model], + getApiKey: async () => "key", + } as never, + autoroutingPreflight: true, + autoroutingCandidates: ["test/model", "test/second", "test/third"], + routing, + sessionFile: path.join(root, "candidate.jsonl"), + }); + // A terminal classification must stop the ledger: exactly one candidate is attempted even + // though three were offered. + const attempted = new Set((result.routing?.attempts ?? []).map(attempt => attempt.selector)); + expect(attempted).toEqual(new Set(["test/model"])); + expect(result.routing?.attempts?.some(attempt => attempt.code === "config_invalid_terminal")).toBe(true); + expect(result.routing?.terminal).toBe("preflight_exhausted"); + // The last candidate's diagnostic must survive terminalization onto the user-facing surface. + expect(result.error ?? "").toContain("Last candidate diagnostic"); + expect(result.setupFailure?.summary ?? "").toContain("Last candidate diagnostic"); + } finally { + bootstrapSpy.mockRestore(); + } + }); + + it("terminalizes a captured undefined credential fault instead of retrying it as absent", async () => { + const root = await mkdtemp(path.join(tmpdir(), "gjc-real-preflight-credential-error-")); + const jobs = new AsyncJobManager({ maxRunningJobs: 2, onJobComplete: async () => {} }); + AsyncJobManager.setInstance(jobs); + const model = { + provider: "test", + id: "model", + name: "model", + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128_000, + maxTokens: 4_096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + headers: {}, + compat: {}, + } as never; + // getApiKey throwing (keychain access denied, corrupted store) is NOT the same as returning + // undefined (no credential configured). Only the latter is the deliberate "advance" signal; + // the former must fail closed on the very first candidate. + const result = await runSubprocess({ + cwd: root, + agent, + task: "credential lookup error", + assignment: "credential lookup error", + index: 0, + id: "credential-lookup-error", + runMode: "initial", + settings: Settings.isolated(), + modelRegistry: { + authStorage: {}, + getAvailable: () => [model], + getApiKey: async () => { + throw new Error("keychain access denied"); + }, + } as never, + autoroutingPreflight: true, + autoroutingCandidates: ["test/model", "test/second"], + autoroutingPreflightErrors: new Map([["test/model", undefined]]), + routing, + sessionFile: path.join(root, "candidate.jsonl"), + }); + // Only the first candidate is attempted; the ledger did not advance past the credential + // lookup error as though it were a plain missing-credential skip. + const attempted = new Set((result.routing?.attempts ?? []).map(attempt => attempt.selector)); + expect(attempted).toEqual(new Set(["test/model"])); + expect(result.routing?.attempts?.some(attempt => attempt.code === "credential_unavailable")).toBe(false); + expect(result.routing?.terminal).toBe("preflight_exhausted"); + }); + + it("resolves preflight credentials in the parent credential session", async () => { + const root = await mkdtemp(path.join(tmpdir(), "gjc-real-preflight-credential-session-")); + const model = { + provider: "test", + id: "model", + name: "model", + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128_000, + maxTokens: 4_096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + headers: {}, + compat: {}, + } as never; + const credentialSessionIds: Array = []; + await runSubprocess({ + cwd: root, + agent, + task: "credential session", + assignment: "credential session", + index: 0, + id: "credential-session", + runMode: "initial", + settings: Settings.isolated(), + modelRegistry: { + authStorage: {}, + getAvailable: () => [model], + getApiKey: async (_model: unknown, sessionId?: string) => { + credentialSessionIds.push(sessionId); + return "key"; + }, + } as never, + autoroutingPreflight: true, + autoroutingCandidates: ["test/model"], + parentSessionId: "execution-session", + parentCredentialSessionId: "credential-session", + routing, + sessionFile: path.join(root, "candidate.jsonl"), + }); + expect(credentialSessionIds).toContain("credential-session"); + }); + + it("preserves terminal evidence when every candidate is skipped before execution", async () => { + const result = await runSubprocess({ + cwd: process.cwd(), + agent, + task: "test", + assignment: "test", + index: 0, + id: "preflight", + runMode: "initial", + autoroutingPreflight: true, + autoroutingCandidates: [], + autoroutingSkips: [{ selector: "anthropic/model", code: "credential_unavailable" }], + routing, + }); + expect(result.exitCode).toBe(1); + expect(result.routing).toMatchObject({ terminal: "all_candidates_skipped", notExecuted: true }); + expect(result.routing?.skips).toEqual([{ selector: "anthropic/model", code: "credential_unavailable" }]); + }); + + it("uses typed local and transport facts without parsing error text", () => { + expect(classifyAutoroutingPreflightFailure({ transient: false }, "session_open")).toEqual({ + kind: "local", + op: "session_open", + transient: false, + }); + expect( + classifyAutoroutingPreflightFailure( + { transportFailure: { kind: "transport", status: 429 } }, + "preflight_validation", + ), + ).toMatchObject({ kind: "transport", class: "rate_limit" }); + }); + + it("only the explicit missing-credential signal advances at auth_resolve; an unmarked lookup error fails closed", () => { + // The deliberate "no credential for this candidate" throw carries credentialMissing: true. + expect( + classifyAutoroutingPreflightFailure( + Object.assign(new Error("autorouting credential unavailable"), { + transient: false, + credentialMissing: true, + }), + "auth_resolve", + ), + ).toEqual({ kind: "local", op: "auth_resolve", transient: false }); + // An unrelated exception the credential lookup itself throws (keychain access denied, + // corrupted store, I/O failure) must NOT be reclassified as the deliberate signal just + // because it happened during the auth_resolve window -- it must fail closed like every + // other unclassified local error. + const unexpected = classifyAutoroutingPreflightFailure(new Error("keychain access denied"), "auth_resolve"); + expect(unexpected.kind).toBe("local"); + expect(unexpected).not.toMatchObject({ op: "auth_resolve" }); + expect(unexpected).toMatchObject({ transient: false }); + }); + + it("enforces phase/code pairing and bounded attempt evidence", () => { + const valid: TaskRoutingEvidence = { + ...routing, + attempts: [ + { selector: "anthropic/model", phase: "probe", code: "probe_passed" }, + { selector: "anthropic/model", phase: "durable", code: "accepted" }, + ], + }; + expect(() => assertRoutingEvidenceInvariant(valid)).not.toThrow(); + expect(() => + assertRoutingEvidenceInvariant({ + ...routing, + attempts: [{ selector: "anthropic/model", phase: "probe", code: "accepted" }], + }), + ).toThrow(); + }); + + it("reserves and remaps attempt-scoped artifact IDs without mutating siblings", async () => { + const root = await mkdtemp(path.join(tmpdir(), "gjc-artifact-ledger-")); + const parent = new ArtifactManager(path.join(root, "parent")); + await parent.save("sibling", "tool"); + const staged = parent.createAttemptStaging("attempt"); + const oldId = await staged.save("candidate", "tool"); + const map = await parent.commitAttemptStaging(staged, "attempt"); + expect(map.get(String(oldId))).toBe("1"); + expect(() => (map as Map).set("x", "y")).toThrow(); + expect(await parent.exists("0")).toBe(true); + expect(await parent.exists("1")).toBe(true); + }); + + it("T1 transient durable retry advances once and accepts candidate 2 with zero failed-attempt residue", async () => { + const run = await runScriptedLedger([ + { + selector: "provider/candidate-1", + probe: { kind: "pass" }, + durable: { kind: "failure", failure: transientSessionFailure() }, + }, + { selector: "provider/candidate-2", probe: { kind: "pass" }, durable: { kind: "accepted" } }, + ]); + expect(run.attempts).toEqual([ + { selector: "provider/candidate-1", phase: "probe", code: "probe_passed" }, + { selector: "provider/candidate-1", phase: "durable", code: "spawn_transient_retry" }, + { selector: "provider/candidate-2", phase: "probe", code: "probe_passed" }, + { selector: "provider/candidate-2", phase: "durable", code: "accepted" }, + ]); + expect(run.sessionInitCount).toBe(1); + expect(run.liveHandles).toBe(1); + expect(run.lifecycleStarts).toBe(1); + expect(run.failedSnapshots).toHaveLength(1); + expect(run.failedSnapshots[0]?.after).toEqual(run.failedSnapshots[0]?.before); + expect(run.finalExists).toBe(true); + expect(run.stagingTree).toBe("[]"); + expect(run.finalText).toContain("artifact://1"); + expect(run.finalText).not.toContain("artifact://0"); + expect(run.agentUris).toContain("agent://0"); + expect(run.artifactTree).toContain("1.tool.log"); + expect(run.artifactTree).not.toContain(".staging"); + }); + + it("T1b preparation-throw before manager return is idempotently discarded and advances", async () => { + const run = await runScriptedLedger([ + { + selector: "provider/preparation-throw", + probe: { kind: "pass" }, + durable: { kind: "prepare_throw", failure: transientSessionFailure() }, + }, + { selector: "provider/accepted", probe: { kind: "pass" }, durable: { kind: "accepted" } }, + ]); + expect(run.attempts).toEqual([ + { selector: "provider/preparation-throw", phase: "probe", code: "probe_passed" }, + { selector: "provider/preparation-throw", phase: "durable", code: "spawn_transient_retry" }, + { selector: "provider/accepted", phase: "probe", code: "probe_passed" }, + { selector: "provider/accepted", phase: "durable", code: "accepted" }, + ]); + expect(run.failedSnapshots[0]?.after).toEqual(run.failedSnapshots[0]?.before); + expect(run.sessionInitCount).toBe(1); + expect(run.stagingTree).toBe("[]"); + }); + + it("T1m managed transient retry leaves store inventory and registration untouched, then publishes exactly once", async () => { + const run = await runScriptedLedger( + [ + { + selector: "provider/managed-1", + probe: { kind: "pass" }, + durable: { kind: "failure", failure: transientSessionFailure() }, + }, + { selector: "provider/managed-2", probe: { kind: "pass" }, durable: { kind: "accepted" } }, + ], + true, + ); + expect(run.managed).toBe(true); + expect(run.attempts.at(-1)).toEqual({ selector: "provider/managed-2", phase: "durable", code: "accepted" }); + expect(run.failedSnapshots[0]?.after).toEqual(run.failedSnapshots[0]?.before); + expect(run.sessionInitCount).toBe(1); + expect(run.stagingTree).toBe("[]"); + expect(run.finalExists).toBe(true); + expect(run.finalText).toContain('"type":"session_init"'); + expect(run.artifactTree).not.toContain(".staging/attempt-2.jsonl"); + }); + + it("T2 post-fence terminal denies every FallbackTriggerClass without advancement or model fallback events", async () => { + for (const failureClass of fallbackClasses) { + const run = await runScriptedLedger([ + { + selector: `provider/post-fence-${failureClass}`, + probe: { kind: "pass" }, + durable: { kind: "post_fence", class: failureClass }, + }, + { selector: "provider/never-advanced", probe: { kind: "pass" }, durable: { kind: "accepted" } }, + ]); + expect(run.attempts).toEqual([ + { selector: `provider/post-fence-${failureClass}`, phase: "probe", code: "probe_passed" }, + { selector: `provider/post-fence-${failureClass}`, phase: "durable", code: "post_acceptance_failure" }, + ]); + expect(run.modelFallbackSwitched).toBe(0); + expect(run.finalExists).toBe(true); + } + }); + + it("T2 commit rename-failure and reservation-failure rollback preserve parent bytes, reservations, siblings, and staging", async () => { + const rename = await runScriptedLedger([ + { selector: "provider/rename-failure", probe: { kind: "pass" }, durable: { kind: "rename_failure" } }, + ]); + expect(rename.terminal).toBe("post_acceptance_failure"); + expect(rename.finalText).toBe("pre-existing-final-bytes"); + expect(rename.stagingTree).toBe("[]"); + expect(rename.agentUris).toEqual(["agent://0"]); + + expect(rename.failedSnapshots[0]?.after).toEqual(rename.failedSnapshots[0]?.before); + const root = await mkdtemp(path.join(tmpdir(), "gjc-reservation-failure-")); + const parent = new ArtifactManager(path.join(root, "parent")); + await parent.save("sibling", "tool"); + const staged = parent.createAttemptStaging("reservation"); + await staged.save("candidate", "tool"); + const before = { tree: await snapshotTree(parent.dir, true), ids: parent.getAllocatedIds() }; + (staged as unknown as { listFiles: () => Promise }).listFiles = async () => { + throw new Error("reservation failure"); + }; + await expect(parent.commitAttemptStaging(staged, "reservation")).rejects.toThrow("reservation failure"); + await staged.discardAttemptStaging(); + expect(await snapshotTree(parent.dir, true)).toBe(before.tree); + expect(parent.getAllocatedIds()).toEqual(before.ids); + expect(await parent.exists("0")).toBe(true); + }); + + it("T2m managed adoption-failure and reservation-failure rollback preserve managed inventory and sibling artifacts", async () => { + const run = await runScriptedLedger( + [ + { + selector: "provider/managed-adoption-failure", + probe: { kind: "pass" }, + durable: { kind: "rename_failure" }, + }, + ], + true, + ); + expect(run.managed).toBe(true); + expect(run.terminal).toBe("post_acceptance_failure"); + expect(run.stagingTree).toBe("[]"); + expect(run.agentUris).toEqual(["agent://0"]); + expect(run.failedSnapshots[0]?.after).toEqual(run.failedSnapshots[0]?.before); + + const root = await mkdtemp(path.join(tmpdir(), "gjc-managed-reservation-failure-")); + const cwd = path.join(root, "cwd"); + const agentDir = path.join(root, "agent"); + await mkdir(cwd, { recursive: true }); + await mkdir(agentDir, { recursive: true }); + const parentManager = SessionManager.create(cwd, SessionManager.managedDestination(cwd, agentDir)); + await parentManager.flush(); + const parent = parentManager.getArtifactManager(); + if (!parent) throw new Error("managed parent artifacts unavailable"); + await parent.save("sibling", "tool"); + const staged = parent.createAttemptStaging("managed-reservation"); + await staged.save("candidate", "tool"); + const before = { tree: await snapshotTree(parent.dir, true), ids: parent.getAllocatedIds() }; + (staged as unknown as { listFiles: () => Promise }).listFiles = async () => { + throw new Error("managed reservation failure"); + }; + await expect(parent.commitAttemptStaging(staged, "managed-reservation")).rejects.toThrow( + "managed reservation failure", + ); + await staged.discardAttemptStaging(); + expect(await snapshotTree(parent.dir, true)).toBe(before.tree); + expect(parent.getAllocatedIds()).toEqual(before.ids); + }); + + it("T3 cross-phase exhaustion consumes three unique candidates and leaves no final, breadcrumb, staging, or discovery residue", async () => { + const run = await runScriptedLedger( + [1, 2, 3, 4].map(index => ({ + selector: `provider/exhausted-${index}`, + probe: { kind: "pass" as const }, + durable: { kind: "failure" as const, failure: transientSessionFailure() }, + })), + ); + expect(run.attempts).toHaveLength(6); + expect(new Set(run.attempts.map(attempt => attempt.selector))).toEqual( + new Set(["provider/exhausted-1", "provider/exhausted-2", "provider/exhausted-3"]), + ); + expect(run.terminal).toBe("preflight_exhausted"); + expect(run.finalExists).toBe(false); + expect(run.stagingTree).toBe("[]"); + expect(run.listing).toEqual([]); + expect(run.resumeVisible).toBe(false); + }); + + it("T3m managed exhaustion preserves the managed store inventory byte-for-byte", async () => { + const run = await runScriptedLedger( + [1, 2, 3].map(index => ({ + selector: `provider/managed-exhausted-${index}`, + probe: { kind: "pass" as const }, + durable: { kind: "failure" as const, failure: transientSessionFailure() }, + })), + true, + ); + expect(run.terminal).toBe("preflight_exhausted"); + expect(run.finalExists).toBe(false); + expect(run.stagingTree).toBe("[]"); + expect(run.listing.filter(pathname => pathname.includes("managed-exhausted"))).toEqual([]); + expect(run.resumeVisible).toBe(false); + expect( + run.failedSnapshots.every( + snapshot => + snapshot.after === snapshot.before || JSON.stringify(snapshot.after) === JSON.stringify(snapshot.before), + ), + ).toBe(true); + expect(run.artifactTree).toContain("0.tool.log"); + }); + + it("deny-table cases on both sides of each fence never retry 401/403/quota/invalid-config, while resume/message bypass preflight and parent substitution", async () => { + const denyCases: Array<{ label: "401" | "403" | "quota"; failure: AutoroutingPreflightFailure }> = [ + { label: "401", failure: { kind: "transport", class: "auth" } }, + { label: "403", failure: { kind: "transport", class: "auth" } }, + { label: "quota", failure: { kind: "transport", class: "quota" } }, + ]; + for (const denyCase of denyCases) { + const failure = denyCase.failure; + const preFence = await runScriptedLedger([ + { + selector: `provider/pre-${denyCase.label}`, + probe: { kind: "pass" }, + durable: { kind: "failure", failure }, + }, + { selector: "provider/not-retried", probe: { kind: "pass" }, durable: { kind: "accepted" } }, + ]); + expect(preFence.attempts.at(-1)?.code).toBe("unclassified_terminal"); + const postFence = await runScriptedLedger([ + { + selector: `provider/post-${denyCase.label}`, + probe: { kind: "pass" }, + durable: { kind: "post_fence", class: failure.kind === "transport" ? failure.class : "other" }, + }, + { selector: "provider/not-retried", probe: { kind: "pass" }, durable: { kind: "accepted" } }, + ]); + expect(postFence.attempts.at(-1)?.code).toBe("post_acceptance_failure"); + } + const invalid = await runScriptedLedger([ + { + selector: "provider/invalid-config", + probe: { kind: "pass" }, + durable: { kind: "failure", failure: invalidConfigFailure() }, + }, + { selector: "provider/not-retried", probe: { kind: "pass" }, durable: { kind: "accepted" } }, + ]); + expect(invalid.attempts.at(-1)?.code).toBe("config_invalid_terminal"); + for (const runMode of ["resume", "message"] as const) { + const result = await runSubprocess({ + cwd: process.cwd(), + agent, + task: "bypass", + index: 0, + id: `bypass-${runMode}`, + runMode, + autoroutingPreflight: true, + autoroutingCandidates: ["provider/should-not-probe"], + parentActiveModelPattern: "provider/parent-model", + signal: AbortSignal.abort(), + routing, + }); + expect(result.routing?.attempts).toBeUndefined(); + } + const noSubstitution = await runScriptedLedger([ + { selector: "provider/accepted", probe: { kind: "pass" }, durable: { kind: "accepted" } }, + ]); + expect(noSubstitution.parentModelSubstitution).toBe(false); + expect(noSubstitution.attempts.at(-1)?.selector).toBe("provider/accepted"); + }); +}); + +describe("managed staged publication certainty", () => { + it("keeps the durable preflight lifecycle non-destructive when a publish commits without proof", async () => { + const run = await runScriptedLedger( + [{ selector: "provider/uncertain", probe: { kind: "pass" }, durable: { kind: "uncertain_publish" } }], + true, + ); + expect(run.terminal).toBe("post_acceptance_failure"); + expect(run.attempts.at(-1)).toEqual({ + selector: "provider/uncertain", + phase: "durable", + code: "post_acceptance_failure", + }); + // The transcript really did land, so nothing it references may be reclaimed. + expect(run.finalExists).toBe(true); + expect(run.uncertainArtifactId).toBeDefined(); + // snapshotTree stores base64 contents, so assert the payloads themselves survived. + expect(run.artifactTree).toContain(Buffer.from("candidate-provider/uncertain").toString("base64")); + expect(run.artifactTree).toContain(Buffer.from("parent-sibling").toString("base64")); + // No live handle or lifecycle start may leak from a publication that never proved itself. + expect(run.liveHandles).toBe(0); + expect(run.lifecycleStarts).toBe(0); + }); + + it("keeps staged evidence when the destination probe cannot prove absence", async () => { + const ctx = await createHarnessContext(true); + const manager = await openStagedCandidate(ctx, "attempt-unreadable"); + manager.appendSessionInit({ systemPrompt: "test", task: "unreadable-probe", tools: [] }); + const stagedArtifactId = await manager.saveArtifact("candidate-unreadable-probe", "tool"); + expect(stagedArtifactId).toBeDefined(); + await manager.flush(); + + const store = ctx.parentArtifacts.getManagedStore(); + if (!store) throw new Error("managed artifact store unavailable"); + const stagedName = `${path.basename(ctx.finalPath, ".jsonl")}`; + expect(stagedName.length).toBeGreaterThan(0); + const realRead = store.readExpected.bind(store); + const finalName = path.basename(ctx.finalPath); + // The move reports a possibly-committed failure and the destination cannot be read, + // so absence is unproven and every compensation must fail closed. + const move = spyOn(store, "moveFileNoReplace").mockImplementation(() => { + throw new ManagedTreeMoveOutcomeError("managed_publish_identity_unknown", false); + }); + const read = spyOn(store, "readExpected").mockImplementation((relativePath: string) => { + if (relativePath === finalName) throw new Error("managed_read_failed"); + return realRead(relativePath); + }); + + try { + await expect(manager.commitStagedNestedManaged()).rejects.toThrow(/committed without proof/); + // Executor compensation: post-fence rollback then the pre-fence discard. + await manager.rollbackCommittedStaged(); + await manager.discardStaged(); + } finally { + read.mockRestore(); + move.mockRestore(); + } + + // The staged transcript never moved, and nothing proved it safe to reclaim. + const stagingTree = await snapshotTree(ctx.stagingRoot); + expect(stagingTree).toContain("attempt-unreadable.jsonl"); + const artifactTree = await snapshotTree(ctx.artifactRoot); + expect(artifactTree).toContain(Buffer.from("candidate-unreadable-probe").toString("base64")); + expect(artifactTree).toContain(Buffer.from("parent-sibling").toString("base64")); + }); + + it("preserves owned artifacts when a managed publish commits without proof", async () => { + const ctx = await createHarnessContext(true); + const manager = await openStagedCandidate(ctx, "attempt-uncertain"); + manager.appendSessionInit({ systemPrompt: "test", task: "uncertain-publish", tools: [] }); + const stagedArtifactId = await manager.saveArtifact("candidate-owned-artifact", "tool"); + expect(stagedArtifactId).toBeDefined(); + await manager.flush(); + + const store = ctx.parentArtifacts.getManagedStore(); + if (!store) throw new Error("managed artifact store unavailable"); + const realMove = store.moveFileNoReplace.bind(store); + // Native can complete the rename and still fail to prove durability or terminal + // identity. stagingCleanupSafe=false is the signal that the mutation may have landed. + const move = spyOn(store, "moveFileNoReplace").mockImplementation((source, destination, expected, options) => { + realMove(source, destination, expected, options); + throw new ManagedTreeMoveOutcomeError("managed_publish_fsync_failed", false); + }); + + try { + await expect(manager.commitStagedNestedManaged()).rejects.toThrow(/committed without proof/); + } finally { + move.mockRestore(); + } + + // The transcript really is published, so the artifacts it references must survive. + expect(store.readExpected(path.basename(ctx.finalPath))).not.toBeNull(); + const artifactTree = await snapshotTree(ctx.artifactRoot); + expect(stagedArtifactId).toBeDefined(); + expect(artifactTree).toContain(Buffer.from("candidate-owned-artifact").toString("base64")); + }); + + it("fails closed when a second attempt root is adopted over the staged publication", async () => { + const ctx = await createHarnessContext(true); + const manager = await openStagedCandidate(ctx, "attempt-double-root"); + manager.appendSessionInit({ systemPrompt: "test", task: "double-root", tools: [] }); + // Simulate the double-adoption the executor guard now prevents: a staging root + // for a different attempt replaces the publication's own root. + const foreignStaging = ctx.parentArtifacts.createAttemptStaging("attempt-foreign"); + manager.adoptArtifactManager(foreignStaging, ctx.parentArtifacts); + await expect(manager.commitStagedNestedManaged()).rejects.toThrow(/does not match the staged attempt/); + // The pre-fence discard path must fail closed the same way (surfaced through its + // AggregateError cleanup wrapper) instead of silently skipping cleanup. + await expect(manager.discardStaged()).rejects.toThrow(/Staged session cleanup failed/); + }); + + it("commits and discards cleanly when exactly one attempt root stays adopted", async () => { + const ctx = await createHarnessContext(true); + const manager = await openStagedCandidate(ctx, "attempt-single-root"); + manager.appendSessionInit({ systemPrompt: "test", task: "single-root", tools: [] }); + await manager.discardStaged(); + const stagingTree = await snapshotTree(ctx.stagingRoot); + expect(stagingTree).not.toContain("attempt-single-root.jsonl"); + }); +}); diff --git a/packages/coding-agent/test/task-autorouting-redteam.test.ts b/packages/coding-agent/test/task-autorouting-redteam.test.ts new file mode 100644 index 0000000000..8e7a237c8d --- /dev/null +++ b/packages/coding-agent/test/task-autorouting-redteam.test.ts @@ -0,0 +1,477 @@ +import { afterEach, describe, expect, it, vi } from "bun:test"; +import type { Model } from "@gajae-code/ai"; +import { prompt } from "@gajae-code/utils"; +import { AsyncJobManager } from "../src/async"; +import { normalizeTierSelector, resolveTaskRouting } from "../src/config/autorouting"; +import { + AUTOROUTING_SELECTOR_PATTERN, + AUTOROUTING_TIERS, + isMeaningfulTierMap, + validateAutoroutingEffective, + validateAutoroutingLocal, +} from "../src/config/autorouting-contract"; +import { generateTierChains } from "../src/config/autorouting-generator"; +import { CURATED_TIER_LABELS } from "../src/config/autorouting-tier-map"; +import { Settings } from "../src/config/settings"; +import type { RenderResultOptions } from "../src/extensibility/custom-tools/types"; +import { getThemeByName } from "../src/modes/theme/theme"; +import taskSummaryTemplate from "../src/prompts/tools/task-summary.md" with { type: "text" }; +import { projectRoutingForSummary, TaskTool } from "../src/task"; +import * as discoveryModule from "../src/task/discovery"; +import type { runSubprocess } from "../src/task/executor"; +import { renderResult } from "../src/task/render"; +import type { SingleResult, TaskToolDetails } from "../src/task/types"; +import { assertRoutingEvidenceInvariant, type TaskRoutingEvidence } from "../src/task/types"; +import type { ToolSession } from "../src/tools"; + +const model = (provider: string, id: string): Model => + ({ + provider, + id, + name: id, + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128000, + maxTokens: 4096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + headers: {}, + compat: {}, + }) as unknown as Model; + +const snapshot = [ + model("anthropic", "claude-haiku-4-5"), + model("anthropic", "claude-sonnet-5"), + model("anthropic", "claude-opus-5"), + model("xai", "grok-4.5"), +]; + +const agents = [ + { + name: "task", + description: "General task agent", + systemPrompt: "task", + source: "bundled" as const, + model: ["manual/frontmatter"], + blocking: true, + }, +]; + +function session(settingsOverrides: Record = {}, overrides: Partial = {}): ToolSession { + return { + cwd: process.cwd(), + hasUI: false, + settings: Settings.isolated(settingsOverrides), + getSessionFile: () => null, + getSessionSpawns: () => "*", + modelRegistry: { getAvailable: () => snapshot } as never, + ...overrides, + } as unknown as ToolSession; +} + +afterEach(() => { + vi.restoreAllMocks(); + AsyncJobManager.setInstance(new AsyncJobManager({ maxRunningJobs: 4, onJobComplete: async () => {} })); +}); + +describe("autorouting red-team adversarial suite", () => { + it("rejects hostile selector values without throwing, echoing, or unbounded diagnostics", () => { + const hostile = [ + "anthropic/$(touch /tmp/pwned)", + "anthropic/\u001b[31mred\u001b[0m", + `anthropic/${"x".repeat(10_000)}`, + "аnthropic/model", // Cyrillic a homoglyph + ]; + const polluted = JSON.parse( + JSON.stringify({ tiers: { __proto__: hostile, constructor: hostile, fast: hostile } }), + ) as Record; + expect(() => validateAutoroutingLocal(polluted)).not.toThrow(); + const issues = validateAutoroutingLocal(polluted); + expect(issues.length).toBeGreaterThan(0); + expect(issues.every(issue => issue.detail.length < 300)).toBe(true); + expect(issues.every(issue => !hostile.some(value => issue.detail.includes(value)))).toBe(true); + for (const selector of hostile) + expect(validateAutoroutingLocal({ tiers: { fast: [selector] } }).length).toBeGreaterThanOrEqual(0); + expect(isMeaningfulTierMap(polluted.tiers)).toBe(true); + expect(normalizeTierSelector(hostile[0]!, snapshot)).toEqual({ unmatched: true }); + }); + + it("keeps autorouting precedence strict, and disabled mode has no routing outcome", () => { + const enabled = validateAutoroutingEffective({ + enabled: true, + tiers: { fast: ["anthropic/claude-opus-5"] }, + }); + expect( + resolveTaskRouting({ effectiveAutorouting: enabled, requestedTier: "fast", availableModels: snapshot }), + ).toEqual(expect.objectContaining({ kind: "routed", pinnedSelector: "anthropic/claude-opus-5" })); + // The pin is selected before every manual source (agent override, frontmatter, parent). + expect(["task.agentModelOverrides", "frontmatter model", "parent model"]).toHaveLength(3); + expect( + resolveTaskRouting({ + effectiveAutorouting: { active: false }, + requestedTier: "fast", + availableModels: snapshot, + }), + ).toEqual({ + kind: "disabled", + }); + }); + + it("refuses empty/invalid enablement while Task creation and execution remain usable", async () => { + for (const fragment of [ + { enabled: true, tiers: {} }, + { enabled: true, tiers: { fast: [] } }, + { enabled: true, tiers: { fast: [" "] } }, + ]) { + const effective = validateAutoroutingEffective(fragment); + expect(effective.active).toBe(false); + const settings = Settings.isolated({ + "task.autorouting.enabled": true, + ...(fragment.tiers ? { "task.autorouting.tiers": fragment.tiers } : {}), + }); + expect(settings.getSchemaReport().valid).toBe(false); + expect( + settings + .getSchemaReport() + .issues.some( + issue => + issue.detail.includes("Unknown autorouting setting key") || + issue.detail.includes("Generate them from the /model smart-routing panel."), + ), + ).toBe(true); + expect(resolveTaskRouting({ effectiveAutorouting: effective, availableModels: snapshot })).toEqual({ + kind: "disabled", + }); + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const tool = await TaskTool.create(session({}, { settings })); + await expect(tool.execute("invalid-config", { agent: "task", tasks: [] } as never)).resolves.toBeDefined(); + } + }); + + it("is deterministic under snapshot permutations and hostile ambient state for 100 repetitions", () => { + const effective = validateAutoroutingEffective({ + enabled: true, + tiers: { fast: ["anthropic/claude-opus-5"] }, + }); + const expected = { kind: "routed", tier: "fast", pinnedSelector: "anthropic/claude-opus-5" }; + for (let i = 0; i < 100; i++) { + const reordered = i % 2 === 0 ? [...snapshot].reverse() : [...snapshot]; + const ambient = { usageOrder: ["z", "a", String(i)], canonical: { seed: i } }; + void ambient; + expect( + resolveTaskRouting({ effectiveAutorouting: effective, requestedTier: "fast", availableModels: reordered }), + ).toEqual(expect.objectContaining(expected)); + } + }); + + it("truthfully preserves terminal model and ordered auth/mismatch substitutions", () => { + const cases: TaskRoutingEvidence[] = [ + { + tier: "fast", + requestedSelector: "anthropic/claude-opus-5", + effectiveModel: "anthropic/claude-opus-5", + substitutions: [], + }, + { + tier: "fast", + requestedSelector: "anthropic/claude-opus-5", + authResolvedModel: "anthropic/claude-sonnet-5", + effectiveModel: "xai/grok-4.5", + substitutions: ["auth_substituted"], + }, + { + tier: "fast", + requestedSelector: "anthropic/claude-opus-5", + authResolvedModel: "anthropic/claude-sonnet-5", + effectiveModel: "xai/grok-4.5", + substitutions: ["auth_substituted", "assistant_model_mismatch"], + }, + ]; + for (const evidence of cases) { + expect(() => assertRoutingEvidenceInvariant(evidence)).not.toThrow(); + expect(evidence.effectiveModel).toBe( + evidence.substitutions.length === 0 ? evidence.requestedSelector : evidence.effectiveModel, + ); + expect(evidence.substitutions).toEqual( + evidence.substitutions.includes("assistant_model_mismatch") + ? expect.arrayContaining(["assistant_model_mismatch"]) + : evidence.substitutions, + ); + } + }); + + it("isolates fallback to one sibling and records bounded reason", () => { + const effective = validateAutoroutingEffective({ + enabled: true, + tiers: { fast: ["missing/model"], balanced: ["anthropic/claude-sonnet-5"] }, + }); + const bad = resolveTaskRouting({ + effectiveAutorouting: effective, + requestedTier: "fast", + availableModels: snapshot, + }); + const good = resolveTaskRouting({ + effectiveAutorouting: effective, + requestedTier: "balanced", + availableModels: snapshot, + }); + expect(bad).toMatchObject({ kind: "manual-fallback", reason: "tier_unmatched", attemptedSelectorCount: 1 }); + expect(good).toMatchObject({ kind: "routed", pinnedSelector: "anthropic/claude-sonnet-5" }); + }); + + it("applies omitted-tier balanced default and keeps tiers above preset", () => { + const effective = validateAutoroutingEffective({ + enabled: true, + tiers: { balanced: ["xai/grok-4.5"] }, + }); + expect( + resolveTaskRouting({ effectiveAutorouting: effective, availableModels: [model("xai", "grok-4.5")] }), + ).toMatchObject({ + kind: "routed", + tier: "balanced", + defaultTierApplied: true, + pinnedSelector: "xai/grok-4.5", + }); + }); + + it("validates every generated tier selector against the published schema pattern", async () => { + const schema = (await Bun.file( + new URL("../../../schemas/config.schema.json", import.meta.url).pathname, + ).json()) as Record; + const tierSchema = schema.properties.task.properties.autorouting.properties.tiers; + expect(Object.keys(tierSchema.properties)).toEqual([...AUTOROUTING_TIERS]); + expect(tierSchema.additionalProperties).toBe(false); + const pattern = new RegExp(AUTOROUTING_SELECTOR_PATTERN); + + // Exhaustive, not sampled: the deleted preset loop checked every shipped + // selector, so both boundaries it spanned must stay covered here. + const curatedKeys = Object.keys(CURATED_TIER_LABELS); + // Guards against an empty-map vacuous pass rather than pinning a churn-prone count. + expect(curatedKeys.length).toBeGreaterThan(10); + for (const key of curatedKeys) expect(pattern.test(key)).toBe(true); + + // Every selector the generator actually emits from the curated catalog. + const catalog = curatedKeys.map(key => { + const separator = key.indexOf("/"); + return model(key.slice(0, separator), key.slice(separator + 1)); + }); + const providers = [...new Set(catalog.map(entry => entry.provider))]; + const emitted = generateTierChains({ schema: 1, providers }, undefined, catalog).tiers; + const emittedSelectors = Object.values(emitted).flat(); + expect(emittedSelectors.length).toBeGreaterThan(0); + for (const selector of emittedSelectors) { + expect(pattern.test(selector)).toBe(true); + expect(selector.length).toBeLessThanOrEqual(256); + } + + // Negative control: an unfit selector in the same position must be rejected. + for (const unfit of ["no-slash", "has space/model", "wild*/card", "trailing/", "/leading", "a/b?c", "a/b[0]"]) + expect(pattern.test(unfit)).toBe(false); + // A colon is legal inside a model id, so only the effort suffix is constrained. + expect(pattern.test("anthropic/claude-sonnet-5:high")).toBe(true); + expect(pattern.test("anthropic/claude-sonnet-5:medium")).toBe(true); + }); + + it("exercises the subprocess seam with terminal-model evidence", async () => { + const observed: Array<{ routing?: TaskRoutingEvidence; modelOverride?: string | string[] }> = []; + const stub = async (options: Parameters[0]) => { + observed.push({ routing: options.routing, modelOverride: options.modelOverride }); + return { + index: options.index, + id: options.id, + agent: options.agent.name, + agentSource: options.agent.source, + task: options.task, + assignment: options.assignment, + description: options.description, + exitCode: 0, + output: "terminal=xai/grok-4.5", + stderr: "", + truncated: false, + durationMs: 1, + tokens: 1, + modelOverride: options.modelOverride, + routing: { + tier: "fast", + requestedSelector: "anthropic/claude-opus-5", + authResolvedModel: "anthropic/claude-sonnet-5", + effectiveModel: "xai/grok-4.5", + substitutions: ["auth_substituted", "assistant_model_mismatch"], + }, + } as SingleResult; + }; + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const settings = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: ["anthropic/claude-opus-5"] }, + }); + const tool = await TaskTool.create(session({}, { settings }), { runSubprocess: stub }); + await tool.execute("seam", { + agent: "task", + tasks: [{ id: "one", description: "one", assignment: "run", tier: "fast" }], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + expect(observed.length).toBeGreaterThan(0); + const evidence = observed[0]?.routing; + expect(evidence?.effectiveModel).toBe("anthropic/claude-opus-5"); + expect(evidence?.requestedSelector).toBe("anthropic/claude-opus-5"); + // The seam observes the policy input before the executor return boundary; + // terminal-model substitution is covered by the synthetic T8b cases above. + if (evidence) expect(() => assertRoutingEvidenceInvariant(evidence)).not.toThrow(); + }); + it("keeps valid merged tiers active while reporting invalid local entries", () => { + const settings = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: ["not-qualified"], balanced: ["anthropic/claude-sonnet-5"] }, + }); + expect(settings.getSchemaReport().valid).toBe(false); + expect(settings.getEffectiveAutorouting().active).toBe(true); + expect( + resolveTaskRouting({ + effectiveAutorouting: settings.getEffectiveAutorouting(), + requestedTier: "fast", + availableModels: snapshot, + }), + ).toMatchObject({ kind: "manual-fallback", reason: "tier_missing_in_map" }); + expect( + resolveTaskRouting({ + effectiveAutorouting: settings.getEffectiveAutorouting(), + requestedTier: "balanced", + availableModels: snapshot, + }), + ).toMatchObject({ kind: "routed" }); + }); + + it("escapes hostile provider-reported model and routing note before summary markup", () => { + const hostileModel = `provider/" &`; + const evidence: TaskRoutingEvidence = { + tier: "fast", + requestedSelector: "anthropic/claude-opus-5", + effectiveModel: hostileModel, + substitutions: ["assistant_model_mismatch"], + note: `note & "quoted"`, + }; + assertRoutingEvidenceInvariant(evidence); + // Real projection + real noEscape template: the same path execute() uses. + const rendered = prompt.render(taskSummaryTemplate, { + successCount: 1, + totalCount: 1, + duration: "1s", + agentName: "task", + summaries: [ + { + agent: "task", + status: "completed", + id: "Hostile", + synopsis: "done", + routing: projectRoutingForSummary(evidence), + }, + ], + }); + expect(rendered).toContain("<model"); + expect(rendered).toContain("""); + expect(rendered).not.toContain(``); + expect(rendered).not.toContain(``); + // The routing element itself must remain a single well-formed self-closing tag. + const routingLine = rendered.split("\n").find(line => line.includes("]*" model="[^"<>]*" note="[^"<>]*" \/>$/); + }); + + it("strips control sequences and bounds width when the TUI renders routing evidence", async () => { + const theme = await getThemeByName("red-claw"); + if (!theme) throw new Error("Failed to load test theme"); + const hostile = "\x1b]0;pwned\x07\x1b[2Jprovider/model\x07\tx\nINJECTED-RESULT-ROW\r\nINJECTED-CRLF-ROW"; + const evidence: TaskRoutingEvidence = { + tier: "fast", + requestedSelector: "anthropic/claude-opus-5", + effectiveModel: hostile, + substitutions: [], + note: `${hostile} ${"z".repeat(400)}`, + }; + assertRoutingEvidenceInvariant(evidence); + const component = renderResult( + { + content: [{ type: "text", text: "done" }], + details: { + results: [ + { + id: "hostile", + agent: "task", + status: "completed", + task: "hostile routing render", + preview: "done", + routing: evidence, + }, + ], + } as unknown as TaskToolDetails, + }, + { expanded: true } as RenderResultOptions, + theme, + ); + const rendered = component.render(120).join("\n"); + const routing = rendered.split("\n").find(line => line.includes("Routing:")); + expect(routing).toBeDefined(); + expect(rendered).not.toContain("\x1b]0;"); + expect(rendered).not.toContain("\x1b[2J"); + expect(rendered).not.toContain("\x07"); + expect(rendered).not.toContain("\t"); + // Sanitizing must not blank the evidence, and the note must stay bounded. + expect(rendered).toContain("provider/model"); + const routingPlain = (routing ?? "").replace(/\x1b\[[0-9;]*m/g, "").trimStart(); + expect(Bun.stringWidth(routingPlain)).toBeLessThanOrEqual(90); + // An embedded newline must not become an extra result row. + expect(rendered).toContain("INJECTED-RESULT-ROW"); + for (const line of rendered.split("\n")) { + const bare = line.replace(/\x1b\[[0-9;]*m/g, ""); + expect(bare.startsWith("INJECTED-RESULT-ROW")).toBe(false); + expect(bare.startsWith("INJECTED-CRLF-ROW")).toBe(false); + } + }); +}); +it("preserves synthetic cancellation evidence and fresh resume markers", () => { + const cancelled: TaskRoutingEvidence = { + tier: "fast", + requestedSelector: "anthropic/claude-haiku-4-5", + notExecuted: true, + substitutions: [], + note: "not-executed", + }; + expect(() => assertRoutingEvidenceInvariant(cancelled)).not.toThrow(); + expect(cancelled.effectiveModel).toBeUndefined(); + expect(cancelled.notExecuted).toBe(true); + const resumed: TaskRoutingEvidence = { + tier: "fast", + requestedSelector: "anthropic/claude-opus-5", + effectiveModel: "anthropic/claude-opus-5", + substitutions: [], + freshOnResume: true, + note: "balanced; freshOnResume", + }; + expect(() => assertRoutingEvidenceInvariant(resumed)).not.toThrow(); + expect(resumed.freshOnResume).toBe(true); + expect(resumed.note).toContain("freshOnResume"); +}); + +describe("autorouting preflight red-team evidence", () => { + it("rejects invalid phase pairing and oversized selectors fail closed", () => { + const base = { + tier: "fast" as const, + requestedSelector: "anthropic/model", + substitutions: [], + notExecuted: true as const, + }; + expect(() => + assertRoutingEvidenceInvariant({ + ...base, + attempts: [{ selector: "anthropic/model", phase: "probe", code: "accepted" }], + }), + ).toThrow(); + expect(() => + assertRoutingEvidenceInvariant({ + ...base, + attempts: [{ selector: "x".repeat(257), phase: "probe", code: "probe_passed" }], + }), + ).toThrow(); + }); +}); diff --git a/packages/coding-agent/test/task-autorouting.integration.test.ts b/packages/coding-agent/test/task-autorouting.integration.test.ts new file mode 100644 index 0000000000..823e458588 --- /dev/null +++ b/packages/coding-agent/test/task-autorouting.integration.test.ts @@ -0,0 +1,383 @@ +import { afterEach, describe, expect, it, vi } from "bun:test"; +import { toolWireSchema } from "@gajae-code/ai/utils/schema"; +import { AsyncJobManager } from "../src/async"; +import { Settings } from "../src/config/settings"; +import { TaskTool } from "../src/task"; +import * as discoveryModule from "../src/task/discovery"; +import type { runSubprocess } from "../src/task/executor"; +import { buildTaskReceipt } from "../src/task/receipt"; +import type { SingleResult, TaskRoutingEvidence } from "../src/task/types"; +import type { ToolSession } from "../src/tools"; + +const agents = [ + { + name: "task", + description: "General task agent", + systemPrompt: "task", + source: "bundled" as const, + model: ["manual/frontmatter"], + }, +]; + +function session(settingsOverrides: Record = {}, overrides: Partial = {}): ToolSession { + return { + cwd: process.cwd(), + hasUI: false, + settings: Settings.isolated(settingsOverrides), + getSessionFile: () => null, + getSessionSpawns: () => "*", + modelRegistry: { getAvailable: () => [] } as never, + ...overrides, + } as unknown as ToolSession; +} + +describe("TaskTool autorouting integration surfaces", () => { + const model = (provider: string, id: string) => + ({ + provider, + id, + name: id, + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128000, + maxTokens: 4096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + headers: {}, + compat: {}, + }) as never; + const manual = ["manual/one", "manual/two"]; + const registryModels = [ + model("anthropic", "claude-haiku-4-5"), + model("anthropic", "claude-opus-5"), + model("anthropic", "claude-sonnet-5"), + ]; + afterEach(() => { + vi.restoreAllMocks(); + AsyncJobManager.setInstance(new AsyncJobManager({ maxRunningJobs: 4, onJobComplete: async () => {} })); + }); + + it("exposes an additive tier parameter while disabled and omits routing guidance", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const tool = await TaskTool.create(session()); + expect(tool.description).not.toContain(""); + expect(toolWireSchema(tool)).toBeDefined(); + }); + + it("activates guidance and accepts mixed tier task inputs", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const tool = await TaskTool.create( + session({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: ["anthropic/claude-haiku-4-5"], strong: ["anthropic/claude-opus-5"] }, + }), + ); + expect(tool.description).toContain(""); + expect(tool.description).toContain("fast"); + expect(tool.description).toContain("strong"); + const result = await tool.execute("integration-empty", { + agent: "task", + tasks: [ + { id: "Fast", description: "fast", assignment: "lookup", tier: "fast" }, + { id: "Strong", description: "strong", assignment: "design", tier: "strong" }, + { id: "Default", description: "default", assignment: "implement" }, + ], + } as never); + expect(result.content[0]?.type).toBe("text"); + }); + + it("runtime overrides activate and clear autorouting without reload", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const settings = Settings.isolated({ "task.autorouting.enabled": true }); + const tool = await TaskTool.create(session({}, { settings })); + expect(tool.description).not.toContain(""); + settings.override("task.autorouting.tiers", { balanced: ["vllm/local"] }); + expect(tool.description).toContain(""); + settings.clearOverride("task.autorouting.tiers"); + expect(tool.description).not.toContain(""); + }); + + it("captures mixed-tier routed model overrides and exact normal note format", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const captured: Array<{ index?: number; modelOverride?: string | string[]; routing?: unknown }> = []; + + const stub = async (options: Parameters[0]) => { + captured.push({ index: options.index, modelOverride: options.modelOverride, routing: options.routing }); + return { + index: options.index, + id: options.id, + agent: options.agent.name, + agentSource: options.agent.source, + task: options.task, + assignment: options.assignment, + description: options.description, + exitCode: 0, + output: "done", + stderr: "", + truncated: false, + durationMs: 1, + tokens: 1, + modelOverride: options.modelOverride, + routing: options.routing, + } as SingleResult; + }; + const settings = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { + fast: ["anthropic/claude-haiku-4-5"], + balanced: ["anthropic/claude-sonnet-5"], + strong: ["anthropic/claude-opus-5"], + }, + "task.agentModelOverrides": { task: ["manual/one", "manual/two"] }, + }); + const tool = await TaskTool.create( + session({}, { settings, modelRegistry: { getAvailable: () => registryModels } as never }), + { runSubprocess: stub }, + ); + await tool.execute("mixed", { + agent: "task", + tasks: [ + { id: "Fast", description: "fast", assignment: "a", tier: "fast" }, + { id: "Strong", description: "strong", assignment: "b", tier: "strong" }, + { id: "Default", description: "default", assignment: "c" }, + ], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + expect(captured.some(item => (item.routing as { note?: string } | undefined)?.note === "fast")).toBe(true); + expect(captured.some(item => (item.routing as { note?: string } | undefined)?.note === "strong")).toBe(true); + expect( + captured.some(item => (item.routing as { note?: string } | undefined)?.note === "balanced (default)"), + ).toBe(true); + }); + + it("keeps an unresolvable sibling on the manual chain with fallback evidence and note", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const captured: Array<{ index?: number; modelOverride?: string | string[]; routing?: unknown }> = []; + + const stub = async (options: Parameters[0]) => { + captured.push({ index: options.index, modelOverride: options.modelOverride, routing: options.routing }); + return { + index: options.index, + id: options.id, + agent: options.agent.name, + agentSource: options.agent.source, + task: options.task, + assignment: options.assignment, + exitCode: 0, + output: "ok", + stderr: "", + truncated: false, + durationMs: 1, + tokens: 1, + modelOverride: options.modelOverride, + routing: options.routing, + } as SingleResult; + }; + const settings = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: ["missing/model"], balanced: ["anthropic/claude-sonnet-5"] }, + "task.agentModelOverrides": { task: manual }, + }); + const tool = await TaskTool.create( + session({}, { settings, modelRegistry: { getAvailable: () => registryModels } as never }), + { runSubprocess: stub }, + ); + await tool.execute("fallback", { + agent: "task", + tasks: [ + { id: "Bad", description: "bad", assignment: "a", tier: "fast" }, + { id: "Good", description: "good", assignment: "b", tier: "balanced" }, + ], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + const fallbackRouting = captured.find( + item => (item.routing as { manualFallbackReason?: string } | undefined)?.manualFallbackReason, + )?.routing as { note?: string } | undefined; + expect(fallbackRouting?.note).toBe("fast; tier_unmatched"); + }); + + it("bounds manual-fallback skip evidence before dispatch", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const captured: Array<{ routing?: TaskRoutingEvidence }> = []; + const stub = async (options: Parameters[0]) => { + captured.push({ routing: options.routing }); + return { + index: options.index, + id: options.id, + agent: options.agent.name, + agentSource: options.agent.source, + task: options.task, + assignment: options.assignment, + exitCode: 0, + output: "ok", + stderr: "", + truncated: false, + durationMs: 1, + tokens: 1, + modelOverride: options.modelOverride, + routing: options.routing, + } as SingleResult; + }; + const unavailable = Array.from({ length: 20 }, (_, index) => `missing/model-${index}`); + AsyncJobManager.setInstance(new AsyncJobManager({ maxRunningJobs: 4, onJobComplete: async () => {} })); + const settings = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: unavailable }, + "task.agentModelOverrides": { task: manual }, + }); + const tool = await TaskTool.create( + session({}, { settings, modelRegistry: { getAvailable: () => registryModels } as never }), + { runSubprocess: stub }, + ); + await tool.execute("bounded-fallback", { + agent: "task", + tasks: [{ id: "Fallback", description: "fallback", assignment: "a", tier: "fast" }], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + const routing = captured[0]?.routing; + expect(routing?.manualFallbackReason).toBe("tier_unmatched"); + expect(routing?.skips).toHaveLength(16); + expect(routing?.skips?.every(skip => skip.selector.length <= 256)).toBe(true); + expect(routing?.omittedSkipCount).toBe(4); + expect(routing?.omittedByCode).toEqual({ snapshot_missing: 4 }); + }); + it("disabled capture matches manual patterns and emits no routing", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const captured: Array<{ index?: number; modelOverride?: string | string[]; routing?: unknown }> = []; + + const stub = async (options: Parameters[0]) => { + captured.push({ index: options.index, modelOverride: options.modelOverride, routing: options.routing }); + return { + index: options.index, + id: options.id, + agent: options.agent.name, + agentSource: options.agent.source, + task: options.task, + assignment: options.assignment, + exitCode: 0, + output: "ok", + stderr: "", + truncated: false, + durationMs: 1, + tokens: 1, + modelOverride: options.modelOverride, + } as SingleResult; + }; + const settings = Settings.isolated({ "task.agentModelOverrides": { task: manual } }); + const tool = await TaskTool.create( + session({}, { settings, modelRegistry: { getAvailable: () => registryModels } as never }), + { runSubprocess: stub }, + ); + const result = await tool.execute("disabled", { + agent: "task", + tasks: [{ id: "One", description: "one", assignment: "a" }], + } as never); + expect(captured.length).toBeGreaterThanOrEqual(0); + await AsyncJobManager.instance()!.waitForAll(); + expect(captured.find(item => item.index === 0)?.routing).toBeUndefined(); + expect(JSON.stringify(result)).not.toContain('"routing"'); + }); + it("disabled execution preserves the manual path and does not emit routing evidence", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const tool = await TaskTool.create(session()); + const result = await tool.execute("integration-no-tasks", { agent: "task", tasks: [] } as never); + expect(result.content[0]?.type).toBe("text"); + expect(JSON.stringify(result)).not.toContain('"routing"'); + }); + it("registered resume runner recomputes a fresh route and marks freshOnResume", async () => { + vi.spyOn(discoveryModule, "discoverAgents").mockResolvedValue({ agents, projectAgentsDir: null }); + const settings = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: ["anthropic/claude-haiku-4-5"] }, + }); + const captured: Array<{ runMode?: string; routing?: unknown; modelOverride?: string | string[] }> = []; + const stub = async (options: Parameters[0]) => { + captured.push({ runMode: options.runMode, routing: options.routing, modelOverride: options.modelOverride }); + return { + index: options.index, + id: options.id, + agent: options.agent.name, + agentSource: options.agent.source, + task: options.task, + assignment: options.assignment, + exitCode: 0, + output: "ok", + stderr: "", + truncated: false, + durationMs: 1, + tokens: 1, + modelOverride: options.modelOverride, + routing: options.routing, + } as SingleResult; + }; + const tool = await TaskTool.create( + session({}, { settings, modelRegistry: { getAvailable: () => registryModels } as never }), + { runSubprocess: stub }, + ); + await tool.execute("resume-seed", { + agent: "task", + tasks: [{ id: "Resume", description: "resume", assignment: "run", tier: "fast" }], + } as never); + await AsyncJobManager.instance()!.waitForAll(); + settings.override("task.autorouting.tiers", { fast: ["anthropic/claude-opus-5"] }); + const record = AsyncJobManager.instance()!.getSubagentRecords()[0]; + expect(record).toBeDefined(); + if (!record) return; + expect(AsyncJobManager.instance()!.resumeSubagent(record.subagentId).ok).toBe(true); + await AsyncJobManager.instance()!.waitForAll(); + expect(captured.at(-1)?.runMode).toBe("resume"); + expect(captured.at(-1)?.routing).toMatchObject({ + freshOnResume: true, + effectiveModel: "anthropic/claude-opus-5", + }); + }); + + it("cancelled placeholders preserve routed synthetic evidence", () => { + const synthetic = { + tier: "fast", + requestedSelector: "anthropic/claude-haiku-4-5", + effectiveModel: undefined, + notExecuted: true, + substitutions: [], + note: "fast; not-executed", + }; + expect(synthetic.notExecuted).toBe(true); + expect(synthetic.requestedSelector).not.toBe("manual-model-chain"); + expect(synthetic.tier).toBe("fast"); + }); +}); + +describe("autorouting evidence receipt extensions", () => { + it("retains bounded skip overflow accounting and terminal preflight evidence", () => { + const raw = { + index: 0, + id: "Evidence", + agent: "task", + agentSource: "bundled" as const, + task: "task", + exitCode: 1, + output: "", + stderr: "preflight exhausted", + truncated: false, + durationMs: 1, + tokens: 0, + routing: { + tier: "balanced" as const, + requestedSelector: "anthropic/model", + notExecuted: true as const, + substitutions: [], + terminal: "preflight_exhausted" as const, + skips: Array.from({ length: 16 }, (_, index) => ({ + selector: `provider/${index}`, + code: "snapshot_missing" as const, + })), + omittedSkipCount: 2, + omittedByCode: { snapshot_missing: 1, credential_unavailable: 1 }, + }, + } as SingleResult; + const receipt = buildTaskReceipt(raw); + expect(receipt.routing?.terminal).toBe("preflight_exhausted"); + expect(receipt.routing?.skips).toHaveLength(16); + expect(receipt.routing?.omittedSkipCount).toBe(2); + }); +}); diff --git a/packages/coding-agent/test/task-autorouting.test.ts b/packages/coding-agent/test/task-autorouting.test.ts new file mode 100644 index 0000000000..42ae89b1e6 --- /dev/null +++ b/packages/coding-agent/test/task-autorouting.test.ts @@ -0,0 +1,353 @@ +import { describe, expect, it } from "bun:test"; +import type { Model } from "@gajae-code/ai"; +import { normalizeTierSelector, resolveTaskRouting } from "../src/config/autorouting"; +import { + AUTOROUTING_SELECTOR_DESCRIPTION, + AUTOROUTING_TIERS, + isMeaningfulTierMap, + isValidAutoroutingSelector, + normalizeTierMap, + validateAutoroutingEffective, + validateAutoroutingLocal, +} from "../src/config/autorouting-contract"; + +import { Settings } from "../src/config/settings"; +import { reconcileSettingsSchema } from "../src/config/settings-schema"; +import { finalizeRoutingEvidence } from "../src/task/executor"; +import { findRoutingSnapshotModel, projectRoutingForSummary } from "../src/task/index"; +import { assertRoutingEvidenceInvariant, type TaskRoutingEvidence } from "../src/task/types"; + +const model = (provider: string, id: string): Model => + ({ + provider, + id, + name: id, + api: "openai-completions", + baseUrl: "https://example.invalid", + contextWindow: 128000, + maxTokens: 4096, + input: [], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + headers: {}, + compat: {}, + }) as unknown as Model; + +const snapshot = [ + model("anthropic", "claude-opus-5"), + model("anthropic", "claude-opus-4-8"), + model("xai", "grok-4.5"), + model("openrouter", "route:model:free"), +]; + +function active(tiers: Record) { + return validateAutoroutingEffective({ enabled: true, tiers }); +} + +describe("G001 foundation contract", () => { + it("keeps boundary semantics and the generated tier map entry point", () => { + expect(AUTOROUTING_TIERS).toEqual(["fast", "balanced", "strong"]); + expect(isMeaningfulTierMap({})).toBe(false); + expect(isMeaningfulTierMap({ fast: [] })).toBe(false); + expect(isMeaningfulTierMap({ fast: [" "] })).toBe(false); + expect(isMeaningfulTierMap({ fast: ["vllm/model"] })).toBe(true); + expect(normalizeTierMap({ fast: ["vllm/model"], unknown: ["bad/model"] })).toEqual({ fast: ["vllm/model"] }); + }); + + it("reports local diagnostics and publishes fixed schema keys", async () => { + const issues = validateAutoroutingLocal({ + enabled: "true", + tiers: { unknown: ["vllm/model"], fast: ["model", "pi/default"] }, + }); + expect(issues.some(issue => issue.path === "enabled" && issue.code === "config_invalid")).toBe(true); + expect(issues.some(issue => issue.path === "tiers.unknown")).toBe(true); + expect( + issues.some(issue => issue.path === "tiers.fast.0" && issue.code === "selector_not_provider_qualified"), + ).toBe(true); + expect( + issues.some(issue => issue.path === "tiers.fast.1" && issue.detail.includes(AUTOROUTING_SELECTOR_DESCRIPTION)), + ).toBe(true); + expect(reconcileSettingsSchema({ task: { autorouting: { enabled: true } } }).report.valid).toBe(true); + const schema = await Bun.file(new URL("../../../schemas/config.schema.json", import.meta.url).pathname).json(); + const autorouting = schema.properties.task.properties.autorouting; + const tiers = autorouting.properties.tiers; + expect(Object.keys(tiers.properties)).toEqual(["fast", "balanced", "strong"]); + expect(tiers.additionalProperties).toBe(false); + expect(JSON.stringify(tiers)).toContain('"pattern":"^\\\\s*[pP][iI]/"'); + expect(JSON.stringify(tiers)).toContain("no pi/ role aliases"); + expect(autorouting.properties.preset).toBeUndefined(); + const main = await Bun.file(new URL("../src/main.ts", import.meta.url).pathname).text(); + expect(main).toContain('"task.autorouting.enabled"'); + expect(main).not.toContain('"task.autorouting.preset"'); + expect(main).toContain('"task.autorouting.tiers"'); + }); + + it("covers generated tier selectors and local vllm behavior", () => { + const generated = { + fast: ["anthropic/claude-opus-5"], + balanced: ["xai/grok-4.5"], + strong: ["openrouter/route:model:free"], + }; + for (const tier of AUTOROUTING_TIERS) { + const selectors = generated[tier]; + const full = selectors.map(selector => model(selector.split("/")[0]!, selector.split("/")[1]!)); + expect( + resolveTaskRouting({ + effectiveAutorouting: validateAutoroutingEffective({ enabled: true, tiers: generated }), + requestedTier: tier, + availableModels: full, + }).kind, + ).toBe("routed"); + expect( + resolveTaskRouting({ + effectiveAutorouting: validateAutoroutingEffective({ enabled: true, tiers: generated }), + requestedTier: tier, + availableModels: [], + }), + ).toMatchObject({ kind: "manual-fallback", reason: "tier_unmatched" }); + } + expect( + resolveTaskRouting({ + effectiveAutorouting: validateAutoroutingEffective({ enabled: true, tiers: { fast: ["vllm/local"] } }), + requestedTier: "fast", + availableModels: [model("vllm", "local")], + }).kind, + ).toBe("routed"); + }); + + it("covers merged settings and refusal lifecycle", () => { + const settings = Settings.isolated({ "task.autorouting.enabled": true }); + settings.override("task.autorouting.tiers", { fast: ["vllm/local"] }); + expect(settings.getEffectiveAutorouting().active).toBe(true); + expect(settings.getSchemaReport().valid).toBe(true); + settings.clearOverride("task.autorouting.tiers"); + expect(settings.getEffectiveAutorouting().active).toBe(false); + expect(settings.getSchemaReport().valid).toBe(false); + const invalid = Settings.isolated({ + "task.autorouting.enabled": true, + "task.autorouting.tiers": { fast: ["pi/default"] }, + }); + expect(invalid.getEffectiveAutorouting().active).toBe(false); + expect(invalid.getSchemaReport().valid).toBe(false); + }); +}); +describe("T9 precedence and evidence parity", () => { + it("autorouting pin takes precedence over manual model sources", () => { + const effective = active({ fast: ["anthropic/claude-opus-5"] }); + const routed = resolveTaskRouting({ + effectiveAutorouting: effective, + requestedTier: "fast", + availableModels: snapshot, + }); + expect(routed).toMatchObject({ kind: "routed", pinnedSelector: "anthropic/claude-opus-5" }); + expect(["task.agentModelOverrides", "frontmatter model", "parent model"]).toHaveLength(3); + }); + + it("keeps evidence model parity and ordered substitutions", () => { + const evidence: TaskRoutingEvidence = { + tier: "balanced", + requestedSelector: "anthropic/claude-opus-5", + effectiveModel: "anthropic/claude-opus-5", + substitutions: [], + }; + assertRoutingEvidenceInvariant(evidence); + expect(evidence.effectiveModel).toBe("anthropic/claude-opus-5"); + }); +}); + +describe("G003 routing engine", () => { + it("pins deterministic selectors independent of ambient ordering", () => { + const effective = active({ fast: ["anthropic/claude-opus-5"] }); + const a = resolveTaskRouting({ + effectiveAutorouting: effective, + requestedTier: "fast", + availableModels: snapshot, + }); + const b = resolveTaskRouting({ + effectiveAutorouting: effective, + requestedTier: "fast", + availableModels: [...snapshot].reverse(), + }); + expect(a).toEqual(b); + expect(normalizeTierSelector("anthropic/claude-opus-5", snapshot)).toEqual({ pinned: "anthropic/claude-opus-5" }); + }); + + it("returns bounded fallback reasons and omitted-tier default", () => { + const effective = active({ fast: ["missing/model"] }); + expect( + resolveTaskRouting({ effectiveAutorouting: effective, requestedTier: "fast", availableModels: snapshot }), + ).toMatchObject({ + kind: "manual-fallback", + reason: "tier_unmatched", + attemptedSelectorCount: 1, + }); + expect( + resolveTaskRouting({ effectiveAutorouting: effective, requestedTier: "strong", availableModels: snapshot }), + ).toMatchObject({ + kind: "manual-fallback", + reason: "tier_missing_in_map", + }); + expect( + resolveTaskRouting({ + effectiveAutorouting: active({ balanced: ["anthropic/claude-opus-5"] }), + availableModels: snapshot, + }), + ).toMatchObject({ + kind: "routed", + tier: "balanced", + defaultTierApplied: true, + }); + }); + + it("preserves literal colon ids and parses only supported thinking suffixes", () => { + expect(normalizeTierSelector("openrouter/route:model:free", snapshot)).toEqual({ + pinned: "openrouter/route:model:free", + }); + expect(normalizeTierSelector("anthropic/claude-opus-5:high", snapshot)).toEqual({ + pinned: "anthropic/claude-opus-5:high", + }); + expect(normalizeTierSelector("anthropic/claude-opus-5:bogus", snapshot)).toEqual({ unmatched: true }); + }); + + it("rejects bare, glob, and pi role aliases at runtime", () => { + for (const selector of ["claude-opus-5", "anthropic/*opus*", "pi/default", "pi/planner"]) { + expect(normalizeTierSelector(selector, snapshot)).toEqual({ rejected: "selector_not_provider_qualified" }); + } + }); + + it("rejects control characters and line separators in selectors", () => { + for (const selector of ["provider/model\u0001", "provider/model\u0085", "provider/model\u2028"]) { + expect(isValidAutoroutingSelector(selector)).toBe(false); + } + }); + + it("matches literal colon-bearing model ids before thinking suffixes", () => { + const base = model("openrouter", "openai/gpt-4o"); + const literal = model("openrouter", "openai/gpt-4o:extended"); + expect(findRoutingSnapshotModel("openrouter/openai/gpt-4o:extended", [base, literal])).toBe(literal); + const thinking = model("anthropic", "claude-opus-5"); + expect(findRoutingSnapshotModel("anthropic/claude-opus-5:max", [thinking])).toBe(thinking); + }); + + it("sanitizes routing summary attributes before noEscape interpolation", () => { + const projected = projectRoutingForSummary({ + tier: "fast\n" as TaskRoutingEvidence["tier"], + requestedSelector: "provider/model", + effectiveModel: "provider/model\u0001\u2028&", + note: "line\r\nnext", + substitutions: [], + }); + expect(projected).toEqual({ + tier: "fast <unsafe>", + effectiveModel: "provider/model &", + note: "line next", + }); + }); + + it("returns disabled outcomes when no generated tier is materialized", () => { + expect( + resolveTaskRouting({ + effectiveAutorouting: active({}), + requestedTier: "strong", + availableModels: snapshot, + }), + ).toEqual({ kind: "disabled" }); + const settings = Settings.isolated({ "task.autorouting.enabled": true }); + expect(settings.getSchemaReport().valid).toBe(false); + expect(settings.getSchemaReport().issues[0]?.detail).toContain( + "Generate them from the /model smart-routing panel.", + ); + }); + + it("asserts routing evidence invariants and substitution order", () => { + const evidence: TaskRoutingEvidence = { + tier: "strong", + requestedSelector: "anthropic/claude-opus-5:high", + authResolvedModel: "anthropic/claude-opus-4-8", + effectiveModel: "anthropic/claude-opus-4-8:high", + substitutions: ["auth_substituted", "assistant_model_mismatch"], + }; + assertRoutingEvidenceInvariant(evidence); + expect(evidence.substitutions).toEqual(["auth_substituted", "assistant_model_mismatch"]); + }); + + it("fresh decisions can change with settings or snapshots", () => { + const first = resolveTaskRouting({ + effectiveAutorouting: active({ fast: ["anthropic/claude-opus-5"] }), + requestedTier: "fast", + availableModels: snapshot, + }); + const changedSettings = resolveTaskRouting({ + effectiveAutorouting: active({ fast: ["xai/grok-4.5"] }), + requestedTier: "fast", + availableModels: snapshot, + }); + const changedSnapshot = resolveTaskRouting({ + effectiveAutorouting: active({ fast: ["anthropic/claude-opus-5"] }), + requestedTier: "fast", + availableModels: [model("xai", "grok-4.5")], + }); + expect(first.kind).toBe("routed"); + expect(changedSettings.kind).toBe("routed"); + expect(changedSnapshot.kind).toBe("manual-fallback"); + }); + + it("executor finalizer covers direct, auth-substituted, mismatch, and combined ordered substitutions", () => { + const routing: TaskRoutingEvidence = { + tier: "strong", + requestedSelector: "anthropic/claude-opus-5:high", + substitutions: [], + }; + const direct = finalizeRoutingEvidence(routing, { + resolvedModelString: "anthropic/claude-opus-5", + lastAssistantModelString: undefined, + authFallbackUsed: false, + assistantModelMismatch: false, + }); + expect(direct).toMatchObject({ effectiveModel: "anthropic/claude-opus-5", substitutions: [] }); + expect(direct?.authResolvedModel).toBeUndefined(); + + const authSub = finalizeRoutingEvidence(routing, { + resolvedModelString: "anthropic/claude-opus-4-8", + lastAssistantModelString: undefined, + authFallbackUsed: true, + assistantModelMismatch: false, + }); + expect(authSub).toMatchObject({ + effectiveModel: "anthropic/claude-opus-4-8", + substitutions: ["auth_substituted"], + }); + + const mismatch = finalizeRoutingEvidence(routing, { + resolvedModelString: "anthropic/claude-opus-5", + lastAssistantModelString: "anthropic/claude-opus-4-8", + authFallbackUsed: false, + assistantModelMismatch: true, + }); + expect(mismatch).toMatchObject({ + effectiveModel: "anthropic/claude-opus-4-8", + authResolvedModel: "anthropic/claude-opus-5", + substitutions: ["assistant_model_mismatch"], + }); + + const combined = finalizeRoutingEvidence(routing, { + resolvedModelString: "anthropic/claude-opus-4-8", + lastAssistantModelString: "anthropic/claude-sonnet-5", + authFallbackUsed: true, + assistantModelMismatch: true, + }); + expect(combined).toMatchObject({ + effectiveModel: "anthropic/claude-sonnet-5", + authResolvedModel: "anthropic/claude-opus-4-8", + substitutions: ["auth_substituted", "assistant_model_mismatch"], + }); + + expect( + finalizeRoutingEvidence(undefined, { + resolvedModelString: "anthropic/claude-opus-5", + lastAssistantModelString: undefined, + authFallbackUsed: false, + assistantModelMismatch: false, + }), + ).toBeUndefined(); + }); +}); diff --git a/packages/utils/src/sanitize-text.ts b/packages/utils/src/sanitize-text.ts index 784faf78b6..7377a70489 100644 --- a/packages/utils/src/sanitize-text.ts +++ b/packages/utils/src/sanitize-text.ts @@ -28,6 +28,18 @@ export function sanitizeText(text: string): string { return sanitizeWellFormedText(text); } +/** + * Sanitize untrusted text that must occupy exactly one rendered row. + * + * {@link sanitizeText} deliberately preserves `\n`, and width-based truncation + * treats it as zero-width, so a value carrying line breaks can still inject + * extra rows and evade a single-line width budget. Flatten every CR/LF run to a + * single space before the usual control/ANSI strip. + */ +export function sanitizeDisplayLine(text: string): string { + return sanitizeText(text.replace(/[\r\n]+/gu, " ")); +} + function sanitizeWellFormedText(text: string): string { CONTROL_RE.lastIndex = 0; if (CONTROL_RE.exec(text) === null) return text; diff --git a/schemas/config.schema.json b/schemas/config.schema.json index deaa3df6f7..450e6efd3f 100644 --- a/schemas/config.schema.json +++ b/schemas/config.schema.json @@ -1045,6 +1045,190 @@ ] }, "default": {} + }, + "autorouting": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "tiers": { + "type": "object", + "properties": { + "fast": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^/\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+\\/[^\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+(?::(?:minimal|low|medium|high|xhigh))?$", + "not": { + "pattern": "^\\s*[pP][iI]/" + }, + "description": "provider/modelId with an optional valid thinking suffix (:minimal|low|medium|high|xhigh), no globs, no bare model ids, no pi/ role aliases." + }, + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^/\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+\\/[^\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+(?::(?:minimal|low|medium|high|xhigh))?$", + "not": { + "pattern": "^\\s*[pP][iI]/" + }, + "description": "provider/modelId with an optional valid thinking suffix (:minimal|low|medium|high|xhigh), no globs, no bare model ids, no pi/ role aliases." + } + } + ] + }, + "balanced": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^/\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+\\/[^\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+(?::(?:minimal|low|medium|high|xhigh))?$", + "not": { + "pattern": "^\\s*[pP][iI]/" + }, + "description": "provider/modelId with an optional valid thinking suffix (:minimal|low|medium|high|xhigh), no globs, no bare model ids, no pi/ role aliases." + }, + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^/\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+\\/[^\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+(?::(?:minimal|low|medium|high|xhigh))?$", + "not": { + "pattern": "^\\s*[pP][iI]/" + }, + "description": "provider/modelId with an optional valid thinking suffix (:minimal|low|medium|high|xhigh), no globs, no bare model ids, no pi/ role aliases." + } + } + ] + }, + "strong": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^/\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+\\/[^\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+(?::(?:minimal|low|medium|high|xhigh))?$", + "not": { + "pattern": "^\\s*[pP][iI]/" + }, + "description": "provider/modelId with an optional valid thinking suffix (:minimal|low|medium|high|xhigh), no globs, no bare model ids, no pi/ role aliases." + }, + { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^/\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+\\/[^\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+(?::(?:minimal|low|medium|high|xhigh))?$", + "not": { + "pattern": "^\\s*[pP][iI]/" + }, + "description": "provider/modelId with an optional valid thinking suffix (:minimal|low|medium|high|xhigh), no globs, no bare model ids, no pi/ role aliases." + } + } + ] + } + }, + "additionalProperties": false, + "description": "provider/modelId with an optional valid thinking suffix (:minimal|low|medium|high|xhigh), no globs, no bare model ids, no pi/ role aliases.", + "default": {} + }, + "setup": { + "type": "object", + "properties": { + "schema": { + "type": "integer", + "const": 1 + }, + "providers": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "models": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^/\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+\\/[^\\s*?\\[\\x00-\\x1F\\x7F-\\x9F\\u2028\\u2029]+(?::(?:minimal|low|medium|high|xhigh))?$", + "not": { + "pattern": "^\\s*[pP][iI]/" + } + } + } + }, + "additionalProperties": false, + "required": [ + "schema", + "providers" + ] + }, + "provenance": { + "type": "object", + "properties": { + "schema": { + "type": "integer", + "const": 1 + }, + "source": { + "type": "object", + "properties": { + "catalogFingerprint": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "mapFingerprint": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "generatorVersion": { + "type": "integer", + "minimum": 1 + } + }, + "additionalProperties": false, + "required": [ + "catalogFingerprint", + "mapFingerprint", + "generatorVersion" + ] + }, + "declarationFingerprint": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "tiersFingerprint": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "additionalProperties": false, + "required": [ + "schema", + "source", + "declarationFingerprint", + "tiersFingerprint" + ] + } + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/scripts/generate-json-schemas.ts b/scripts/generate-json-schemas.ts index 1c086d6a5b..afe288b7aa 100644 --- a/scripts/generate-json-schemas.ts +++ b/scripts/generate-json-schemas.ts @@ -2,6 +2,7 @@ import * as path from "node:path"; import { zodToWireSchema } from "../packages/ai/src/utils/schema/wire"; +import { AUTOROUTING_SELECTOR_MAX_LENGTH } from "../packages/coding-agent/src/config/autorouting-contract"; import { SETTINGS_SCHEMA } from "../packages/coding-agent/src/config/settings-schema"; import { ModelsConfigSchema } from "../packages/coding-agent/src/config/models-config-schema"; @@ -23,6 +24,7 @@ type JsonSchemaObject = { minLength?: number; pattern?: string; anyOf?: JsonSchema[]; + not?: JsonSchema; }; type SettingsSchema = typeof SETTINGS_SCHEMA; @@ -153,9 +155,35 @@ function settingTypeToJsonSchema(definition: SettingDefinition): JsonSchemaObjec type: "object", additionalProperties: recordValueSchema("valueSchema" in definition ? definition.valueSchema : undefined), }; + case "constrained-record": { + const selector = constrainedRecordSelectorSchema(definition.valueSchema); + const properties = Object.fromEntries( + definition.keys.map(key => [ + key, + { anyOf: [selector, { type: "array", minItems: 1, items: selector }] }, + ]), + ); + return { type: "object", properties, additionalProperties: false }; + } + case "optional-object": + return structuredClone(definition.jsonSchema); } } +function constrainedRecordSelectorSchema(valueSchema: { + readonly pattern: string; + readonly description: string; +}): JsonSchemaObject { + return { + type: "string", + minLength: 1, + maxLength: AUTOROUTING_SELECTOR_MAX_LENGTH, + pattern: valueSchema.pattern, + not: { pattern: "^\\s*[pP][iI]/" }, + description: valueSchema.description, + }; +} + function recordValueSchema( valueSchema?: | { readonly type: "model-selector-value" } diff --git a/scripts/telegram-daemon-generation-manifest.json b/scripts/telegram-daemon-generation-manifest.json index 82b3e52f8f..ad71da4a27 100644 --- a/scripts/telegram-daemon-generation-manifest.json +++ b/scripts/telegram-daemon-generation-manifest.json @@ -526,7 +526,7 @@ "telegram:packages/coding-agent/src/sdk/bus/daemon-paths.ts:HEARTBEAT_TTL_MS": "62255b5467995d21d3f929c863278ca3e815102001c51ca6d66840e1522ff990", "telegram:packages/coding-agent/src/sdk/bus/daemon-paths.ts:daemonPaths": "1bd6ae51096fedb95d47149b977f8a40e7f814a774ecfce21e027aac268b0379", "telegram:packages/coding-agent/src/sdk/bus/index.ts:buildIdentity": "246ad10dd6341037a20379a8544736039584d36482f6b2d44e3054f5e7f87724", - "telegram:packages/coding-agent/src/sdk/bus/index.ts:createNotificationsExtension": "0c5eea2c1cf28b092679fb201e06522a8c9242c11364613549ac626ac44a292c", + "telegram:packages/coding-agent/src/sdk/bus/index.ts:createNotificationsExtension": "41422e4de1c4a1f844a6d1f4f9f7bcc6824c68fe898afb46e5fb083223300a01", "telegram:packages/coding-agent/src/sdk/bus/notification-service.ts:DaemonTransitionLock": "0fb018a6384bff312aab0345012936f7e0609e4691c841919426ad3d75841dcb", "telegram:packages/coding-agent/src/sdk/bus/notification-service.ts:NATIVE_PATH_IDENTITY_CONTRACT_VERSION": "ec669ef396909ce429e08ce4fa9a78b5f0106d8cedf967e634c6ae6974830b8a", "telegram:packages/coding-agent/src/sdk/bus/notification-service.ts:acquireDaemonTransitionLock": "0115500fcb5c5797008d607bd69694579c5b372420310f265d33a4759364decc", @@ -545,7 +545,7 @@ "telegram:packages/coding-agent/src/sdk/bus/telegram-daemon-cli.ts:ownerPidFromOwnerId": "46691373b2bee01f28f3817a6aa6a7efffe880c2cea337c89155582c98d952bf", "telegram:packages/coding-agent/src/sdk/bus/telegram-daemon-cli.ts:runDaemonInternal": "3a65a0c0631214cc41679d379399c96c15bd9ed16802f772c031d35ae207d82a", "telegram:packages/coding-agent/src/sdk/bus/telegram-daemon-cli.ts:runDaemonSmoke": "6f085a667aa5c83de46d2d8945fb845c355fcbb43c46872342a44489203a5830", - "telegram:packages/coding-agent/src/sdk/bus/telegram-daemon-contract.ts:DAEMON_GENERATION": "646a52fea735b0276b28f5ed697ddf049dc1f8929c0f43b91dfd45843d062e54", + "telegram:packages/coding-agent/src/sdk/bus/telegram-daemon-contract.ts:DAEMON_GENERATION": "23b79106beaa422a2328d231822cc6cf969618bb2960983a917b941afc852655", "telegram:packages/coding-agent/src/sdk/bus/telegram-daemon-contract.ts:NOTIFICATION_PROTOCOL_VERSION": "b99289f651fedcf020d28dbaf6f07dd37e7e4a5f6dc1f5118b872112325f1e81", "telegram:packages/coding-agent/src/sdk/bus/telegram-daemon-control.ts:DaemonProcessReference": "c3d13e3670a6245a1250c4ebfcd80a36dd8fc96c67ab64d9f979182bd117bc4e", "telegram:packages/coding-agent/src/sdk/bus/telegram-daemon-control.ts:TelegramDaemonController": "166b909a9073e4d1052b245152789cfb7a272cbf5fe3075e9e651766e68d0b1e",