diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 742130067..b9d8e01a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -897,7 +897,7 @@ jobs: - name: Test the ACP surface run: | scripts/ci/run-scoped-suite.sh "acp server" acp,runner,tinymemory server::acp - scripts/ci/run-scoped-suite.sh "acp run turn" acp,runner,tinymemory harness::acp_run_turn + scripts/ci/run-scoped-suite.sh "acp run turn" acp,runner,tinymemory harness::acp::run_turn # Issue #788: Chargebee billing — the REST layer (`chargebee::`) and the # toolbelt bridge (`harness::chargebee`) in one filter, which selects both. @@ -1013,13 +1013,13 @@ jobs: # entry that decides whether an agent can reach them at all. An unrun # namespace pin is the wrong thing to have never executed. - name: Test tool-belt contract (openhuman, mcp, telegram, media) - run: scripts/ci/run-scoped-suite.sh "tool-belt contract" openhuman,mcp,telegram,media harness::build::tests + run: scripts/ci/run-scoped-suite.sh "tool-belt contract" openhuman,mcp,telegram,media harness::built_in::build::tests # The `media` toolbelt's other half: the three tool names and their single # `media` namespace. Same feature set as the step above, so this costs a # filtered re-run of an already-compiled tree and no fourth resolution. - name: Test the media toolbelt - run: scripts/ci/run-scoped-suite.sh "media toolbelt" openhuman,mcp,telegram,media harness::toolbelt + run: scripts/ci/run-scoped-suite.sh "media toolbelt" openhuman,mcp,telegram,media harness::built_in::toolbelt # The `mcp` OAuth-flow state, which the belt filter above never reached. # `app::types` holds two gated tests — a parked flow is SINGLE-USE (a @@ -1079,10 +1079,10 @@ jobs: # the ACP step above states: the first bare argument is the filter and a # second would narrow the selection to nothing. - name: Test Composio tenant isolation - run: scripts/ci/run-scoped-suite.sh "composio isolation" openhuman,tinycortex,chargebee,paypal,composio harness::composio::isolation_tests + run: scripts/ci/run-scoped-suite.sh "composio isolation" openhuman,tinycortex,chargebee,paypal,composio harness::built_in::composio::isolation_tests - name: Test the Composio ops helpers - run: scripts/ci/run-scoped-suite.sh "composio ops helpers" openhuman,tinycortex,chargebee,paypal,composio harness::composio::ops_helper_tests + run: scripts/ci/run-scoped-suite.sh "composio ops helpers" openhuman,tinycortex,chargebee,paypal,composio harness::built_in::composio::ops_helper_tests # Issue #820 — which connected account an agent acts as. A third narrow # filter for the same reason the two above are narrow, and the same reason @@ -1093,7 +1093,7 @@ jobs: # the negative half is the one protecting every existing single-account # company from having its account resolution changed. - name: Test which Composio account an execute acts as - run: scripts/ci/run-scoped-suite.sh "composio account choice" openhuman,tinycortex,chargebee,paypal,composio harness::composio::live::live_tests + run: scripts/ci/run-scoped-suite.sh "composio account choice" openhuman,tinycortex,chargebee,paypal,composio harness::built_in::composio::live::live_tests # Issue #820 — a fourth narrow filter, and the first outside `harness::`. # The console-plane half of the same decision: the grouping the choice is diff --git a/docs/spec/README.md b/docs/spec/README.md index 2a2a2181d..cf6e1e671 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -90,6 +90,8 @@ L0 Substrate api.tinyhumans.ai, openhuman-core, tiny.place, filesystem | [runtime/ports-runs.md](runtime/ports-runs.md) | `RunStore`: attempts and their traces | | [runtime/events.md](runtime/events.md) | `CompanyEvent` vocabulary + journal correlation rules | | [runtime/manifest.md](runtime/manifest.md) | `company.toml` schema, `agents.toml` compatibility | +| [runtime/harnesses.md](runtime/harnesses.md) | Named execution engines: `built_in` vs `acp`, transports, per-agent binding | +| [runtime/providers.md](runtime/providers.md) | Inference providers, dual-mode OpenRouter, per-harness credentials | | [runtime/globals.md](runtime/globals.md) | The global baseline every company gets: agents, workflows, skills, the starting tool belt, and `[globals].disable` | | [runtime/lifecycle.md](runtime/lifecycle.md) | Company state machine and durability | | [runtime/planning.md](runtime/planning.md) | The Planning station: pass contract, prerequisite verdicts, boot sweep | diff --git a/docs/spec/runtime/README.md b/docs/spec/runtime/README.md index 07f3ec37e..f99a5b972 100644 --- a/docs/spec/runtime/README.md +++ b/docs/spec/runtime/README.md @@ -46,7 +46,8 @@ Supporting docs: - [artifacts.md](artifacts.md) — what makes something a deliverable: the explicit-publish rule, `(task, source)` identity, body caps and reference bodies, and the single follow-up nudge -- [manifest.md](manifest.md) — `company.toml` schema +- [manifest.md](manifest.md) — `company.toml` schema, + with [manifest-semantics.md](manifest-semantics.md) for each key's behaviour - [globals.md](globals.md) — the global baseline: the agents, workflows, skills and starting tool belt every company gets whichever vertical it started from, how a company supersedes or disables one, and why provenance is persisted diff --git a/docs/spec/runtime/agents.md b/docs/spec/runtime/agents.md index b2f0ad631..557bbcd6f 100644 --- a/docs/spec/runtime/agents.md +++ b/docs/spec/runtime/agents.md @@ -69,6 +69,9 @@ layout first. role = "Copywriter" # required description = "Write ads and campaign copy." tier = "reasoning" # cognition hint; never selects a model +harness = "deep" # which [[harness]] runs this agent's + # turns — see harnesses.md. Omitted + # means the company's default harness. tools = ["docs.*", "mcp:notion"] # grant globs — see tools.md delegates_to = ["creative"] # desks this agent may hand work to budget_usd_daily = 5.0 # per-agent daily cap @@ -90,6 +93,19 @@ ledgers = [ # per-agent ledger access (omit for can_declare_ledgers = false # may this agent `define_ledger`? default true ``` +### `tier` versus `harness` + +They answer different questions and are deliberately separate fields. + +`tier` names a **workload** (`reasoning`, `vision`, …) and is resolved against +whatever provider the agent's harness turns out to use. `harness` names the +**engine and the credential**. So an agent keeps its tier when it moves between +harnesses, and two agents sharing a tier on different harnesses run on different +models — which is the point of naming more than one. + +Naming a harness the company does not declare is a validation error, reported +against both the agent and the id. Naming none is not: every roster written +before `[[harness]]` existed binds nobody, and all of them keep working. ### `context` write access A bare string in `context` is read-only — routed into the prompt, nothing diff --git a/docs/spec/runtime/credentials.md b/docs/spec/runtime/credentials.md index 53369f77c..b8a568f53 100644 --- a/docs/spec/runtime/credentials.md +++ b/docs/spec/runtime/credentials.md @@ -189,8 +189,20 @@ one account into a broken toolkit: when the console revokes an account, and when whatever credential the company's *declared provider* wants — an OpenRouter `sk-or-…`, a raw BYOK token, an `openai_compatible` key. It is provider-scoped, not an identity, and handing it to the TinyHumans backend would present one -vendor's credential to another. The two coincide only when the declared provider -is `managed`, and even then they are the same value for different reasons. +vendor's credential to another. + +Since `managed`'s removal the two never coincide, which makes the separation +cleaner rather than looser. A company holding **no** inference key rides the +subscription on the platform's own credential — resolved from this host's +identity, not from `inference/key` — and a company that sets one is naming an +OpenRouter account that has nothing to do with TinyHumans. See +[providers.md](providers.md). + +There is one such slot **per harness**: the default harness keeps the flat +`inference/key`, and every named one uses `harness//inference/key`. The +asymmetry is deliberate — the `SecretStore` has no rename, so namespacing the +default too would orphan the stored credential of every company already +running. ## What this does not cover diff --git a/docs/spec/runtime/harnesses.md b/docs/spec/runtime/harnesses.md new file mode 100644 index 000000000..46e7cfdde --- /dev/null +++ b/docs/spec/runtime/harnesses.md @@ -0,0 +1,227 @@ +# Harnesses + +*What actually runs an agent's turn, and how a company picks.* + +Terms: [glossary](../glossary.md). The models a harness talks to are +[providers.md](providers.md); the roster it runs is [agents.md](agents.md). + +--- + +## What a harness is + +A **harness** is one answer to "what runs this agent's turn". A company declares +a named set of them and binds each teammate to one, so a single roster can span +a cheap model, an expensive one, and the operator's own coding CLI. + +Two kinds ship: + +| kind | what runs the turn | credential | +|---|---|---| +| `built_in` | the embedded OpenHuman/tinyagents loop, in this process | its own `[harness.inference]` | +| `acp` | an external agent over the Agent Client Protocol | the agent's own | + +`built_in` is the default and the only kind that consults +[providers.md](providers.md). An ACP agent already holds a credential — that is +the point of it — so it needs nothing from us. + +### The case this exists for + +A desktop company with **no key at all**. The operator has Claude Code installed +and signed in; OpenCompany drives it over ACP against their existing +subscription. Nothing to configure on first run, which is a materially different +product from one that opens on a credential form. + +The same seam serves two more things at no extra cost: reverse dispatch (a cloud +host hands work to a runner on someone's machine, which is an ACP agent as far +as this is concerned) and any other harness that speaks the protocol. + +--- + +## Declaring harnesses + +```toml +[[harness]] +id = "embedded" +kind = "built_in" +default = true + +[harness.inference] # attaches to the entry above +provider = "openrouter" + +[[harness]] +id = "deep" +kind = "built_in" + +[harness.inference] +provider = "openrouter" +api_key_secret = "harness/deep/inference/key" +models = { "reasoning-v1" = "" } + +[[harness]] +id = "my_laptop" +kind = "acp" + +[harness.acp] +transport = "local" +agent = "claude" +``` + +`[harness.inference]` and `[harness.acp]` attach to the **most recently +declared** `[[harness]]`. That is ordinary TOML array-of-tables sub-table +syntax, but it is easy to misread as a company-level section, so it is worth +reading twice. + +### Binding an agent + +```toml +# agents/researcher.toml +role = "Researcher" +harness = "deep" +``` + +Inline `[[agent]]` entries take the same field. An agent naming no harness runs +on the one marked `default = true`. + +### The implicit harness + +A company with **no `[[harness]]` block** gets one implicit `built_in` harness, +marked default, inheriting the company-level `[inference]`. Every bundle under +`companies/` and every existing tenant lands here, so named harnesses are purely +additive: nothing has to be rewritten to keep working. + +Read harnesses through `CompanyManifest::effective_harnesses`, never the bare +`harnesses` field. A company that declares none still runs on a harness, and a +caller reading the raw field would see an empty list and conclude it has no +engine, which is never true. + +--- + +## Validation + +`CompanyManifest::validate` rejects, in prosumer language: + +- a duplicate, empty, or non-snake_case `id` +- zero or more than one `default = true`, naming the candidates either way +- an agent naming a harness nothing declares, naming what *is* declared +- `[harness.inference]` on an `acp` kind, or `[harness.acp]` on a `built_in` one +- `transport = "local"` with no `agent`, or naming a `runner`; and the reverse + for `transport = "runner"` + +A section on the wrong kind is an **error, not an ignored key**. This is the +same rule [agents.md](agents.md) applies to a bundle carrying both roster forms, +and for the same reason: a silently discarded declaration stays invisible until +the thing it configured misbehaves, and "my model setting does nothing" is an +expensive way to discover that `[harness.inference]` needs `kind = "built_in"`. + +--- + +## ACP transports + +```toml +[harness.acp] +transport = "local" # spawn an agent on this machine +agent = "claude" # claude | codex | goose + +[harness.acp] +transport = "runner" # reach one that dialed in +runner = "stevens_laptop" +``` + +**A remote runner is a transport, not a third kind.** +`src/runner/dispatch.rs::RunnerDispatch` already implements the same `AcpAgent` +port the local subprocess does, so the only thing that differs is how bytes +reach the agent. Modelling it as a third kind would add a resolution path that +resolves to the same place. + +The transports differ in where they live, which is why `AcpAgent` is a **port** +rather than an ACP client in the host crate: a subprocess over stdio belongs to +the desktop shell, a WebSocket to the runner lane. The same inversion the +storage ports use. + +### Readiness + +For `transport = "local"`, the desktop probes four states rather than two: + +| state | what to do | +|---|---| +| `NotInstalled` | install it | +| `NotSignedIn` | sign in | +| `Ready` | — | +| `SpawnFailed` | read the reason | + +**Installed but not signed in** is the most common state on a fresh machine, and +it looks identical to "not installed" if all you check is `which`. The fixes are +completely different, so collapsing them tells someone to do the wrong thing. + +Sign-in is probed by looking for the harness's credential file, not by running +it: asking a harness whether it is logged in means starting it, which is slow on +a list refreshed whenever a settings pane opens, and for some prompts +interactively. The probe can be wrong in one direction — a stale credential +reads as signed in — and that is the acceptable direction, because the failure +then surfaces on first use with the harness's own message, which is more +accurate than anything guessed. + +--- + +## Routing + +`HarnessRouter` (`src/harness/router.rs`) holds one `RunTurn` per declared +harness and forwards each call to the one its agent names. `RunTurn` already +carried `agent_id` on all three of its methods, so the dispatch point always +existed — nothing had ever varied on it. + +The lanes are built at runtime-build time by `harness::lanes::build`, and +`HarnessBrain` routes through them. **A company declaring one harness (or none) +builds no router at all** — `run_turn()` hands back the single lane directly, so +the overwhelmingly common path is byte-identical to what it was. + +Each `built_in` lane gets its own `HarnessPool` and its own `HarnessDeps`, +differing in exactly two fields: the provider (scoped to that harness's config +and credential slots) and `serves`, which narrows the pool to the agents bound +to it. That narrowing is what makes one-pool-per-harness affordable — without +it, a ten-agent roster across three harnesses would stand up thirty live agents +to use ten. + +All three methods route. A method forwarding to a fixed engine would send +*dispatched card* turns to the wrong model while operator chat looked correct. + +### A harness with no engine fails the turn + +A harness can be declared, valid, and still have no engine. Today that is every +`acp` harness on a server build: the transports live in the desktop shell (a +stdio subprocess) and the runner lane (a socket), and neither is wired into the +server. Those turns fail, naming the harness and the fix. + +They MUST NOT fall back to another harness's engine. That is the worst outcome +available: the turn would succeed, on a model and a credential nobody chose, and +the only evidence would be a billing line. + +--- + +## What a harness does not decide + +- **`[brain].mode`** (`hosted` | `sidecar`) is a separate axis. It selects the + cognition seam *within* the built-in harness. +- **Tools, policy, budgets, desks.** All company- or agent-scoped, and unchanged + by which engine runs the turn. An ACP agent is still subject to the company's + approval policy. +- **Which model an agent's `tier` means.** A tier names a workload and is + resolved against whatever provider its harness turns out to use, so an agent + keeps its tier when it moves between harnesses. See + [providers.md](providers.md). + +--- + +## Implementation map + +| concern | where | +|---|---| +| manifest types, kind/transport vocabularies | `src/company/types.rs` | +| validation, `effective_harnesses`, `harness_for` | `src/company/manifest.rs` | +| per-agent dispatch | `src/harness/router.rs` | +| building the lanes at boot | `src/harness/lanes.rs` | +| the built-in engine | `src/harness/built_in/` | +| the ACP `RunTurn` and its port | `src/harness/acp/run_turn.rs` | +| local transport: discovery, spawn, codec | `src-tauri/src/acp/` | +| runner transport | `src/runner/dispatch.rs` | +| per-harness roster narrowing | `HarnessDeps::serves` | diff --git a/docs/spec/runtime/manifest-semantics.md b/docs/spec/runtime/manifest-semantics.md new file mode 100644 index 000000000..a2b2ab85a --- /dev/null +++ b/docs/spec/runtime/manifest-semantics.md @@ -0,0 +1,362 @@ +# Company Manifest: Semantics + +The behaviour each `company.toml` key and table actually has, beyond the +schema sketch in [manifest.md](manifest.md). Split out from that file to keep +each page under the 500-line cap. + +## Semantics + +- **`[company]`** becomes the seed of the [Charter](../company-brain/charter.md). + `handle` is only used when `[place].discoverable = true`. +- **`[[agent]]`** entries define the Roster — or, equivalently, one + `agents/.toml` file per teammate under the company bundle, carrying the + same keys with the filename as the id. The two forms are **exclusive**: + declaring both is a validation error rather than a precedence rule, because + either precedence would silently discard teammates somebody wrote down. Full + schema, including `prompt` / `prompt_files` / `context` / `classes`, in + [runtime/agents.md](agents.md). + + `tier` is a hint the brain may use when delegating; it never selects a model + (the backend maps tiers to SKUs). `tools` and `budget_usd_daily` intersect + with the company-wide `[tools].allow` and `[budget]` — the most restrictive + wins. Tool grants resolve through **three** levels, + `[tools].allow ∩ [[group_chat]].tools ∩ [[agent]].tools`, every one of them + narrow-only and an empty one a pass-through; see + [runtime/tools.md](tools.md), which also covers why an empty grant list means + "inherit" rather than "nothing". + + **`delegates_to`** (issue #176) is the one per-agent key that is *not* a + narrowing of a company-wide list: it is an **opt-in**. Empty — the default, + and every manifest written before it existed — means the agent carries no + delegation tool at all, which is how a dispatched desk agent has always + behaved. Naming one or more desks wires exactly two tools onto it, + `spawn_task` and a `delegate_to_desk` narrowed to those desks, so a desk lead + can pull a specialist in for one slice instead of handing the whole request + back to the orchestrator. + + It takes **desk** ids or names (`[[group_chat]]` entries), never teammate + ids — desks are the address space `delegate_to_desk` already resolves + against — and `"*"` means every desk the company has. An entry that names no + declared desk fails validation, because at runtime it would fail silently: + the member would carry the tool and every call would be refused. + + It never confers the orchestrator's *authority* — `assign_task`, + `review_task`, `add_agent`, `query_company`, `run_workflow`, + `create_workflow`, and the #661 workflow-admin trio (`read_workflow`, + `update_workflow`, `delete_workflow`) stay orchestrator-only. A member gets what it needs to pass + a slice on and to leave the rest tracked, and nothing more. + + Three runtime guards bound what it can do, all enforced at the tool boundary + in the member's own turn rather than by which tools were wired (belts are + cached per roster, so a tool cannot be withheld from one turn): + + - **Depth** — `[tools].max_delegation_depth`, below. + - **Cycles** — a hand-off to a desk already on the current chain (A→B→A), or + to the desk the caller itself leads, is refused. + - **Allowlist** — a target outside `delegates_to` is refused, and the refusal + names the desks the member *can* reach so it can retry in the same turn. + + Each refusal reaches both the model and the board: the run trail carries it + verbatim, and a refused hand-off is recorded on the dispatched card's note, + so the operator reads the fact rather than inferring it from an absence. + + The per-turn fan-out cap (three delegations) applies **per level**, not per + message — each turn starts against an empty queue. + + **`budget_usd_daily`** (enforced since issue #304 — before that it was + validated, stored and displayed, but nothing read it) caps one teammate's + spend over the **UTC calendar day**, resetting at `00:00Z` — the same + boundary `[plan]`'s daily token budget uses. Spend is re-read from the usage + meter on every check, so a restart mid-day resumes against the real figure + rather than a fresh zero. + + It covers **metered, attributed** spend: inference turns and priced tool + calls (`web_search`, `media_generate_*`), plus any tool call that declares an + `amount_usd`. Two behaviours at the cap: + + - **Dispatch is refused** for that teammate, before any model call, with a + notice naming the cap and the reset. The rest of the company keeps running. + - **A priced tool call parks for approval** rather than being denied. + Approving it runs that one call and nothing more; the cap is not raised. + Free reads and sends are unaffected — a spend cap caps spend. + + Known limits, stated rather than papered over: + + - **Executed x402 payments escape the counter.** Ledger entries carry no + agent, so there is nothing to attribute them to. What *is* covered is the + pre-flight case: a call declaring an `amount_usd` that would breach the + remaining budget parks before the money moves. Company-wide payment + spending is governed by `[budget].monthly_usd`, which is enforced on the + economy path. + - **Turn-boundary overshoot.** A turn or call that starts under the cap can + finish over it; the overshoot is bounded by one call. There is no + reservation ledger in v1 — the same documented window `[plan]` carries. + - **Unreadable spend fails differently at each layer, deliberately.** With no + meter or a failing meter, dispatch **runs** (bricking a teammate's + cognition with no operator recourse is worse than a day of overspend), + while a priced tool call **parks** (a human can wave that one through). + Since issue #343 the manifest value is a **default, not the last word**. An + admin can set, change or clear a teammate's cap from the console + (`PUT`/`DELETE …/team/{agentId}/budget`); the override is stored on the + company record, wins over `budget_usd_daily` everywhere the cap is read + ([`CompanyRecord::effective_budget`] is the single reconciliation point), and + takes effect on the teammate's **next dispatch** — no restart, no redeploy. + That matters most on a hosted tenant, where `company.toml` is baked into the + container image and an operator has no way to edit it. Three stored states, + kept deliberately distinct: no override (the manifest applies), a cap of `x` + (`0` included, meaning "may not spend"), and an explicit "uncapped" that beats + a manifest cap. `DELETE` drops the override so the manifest applies again, + which no `PUT` body can express. Writes are admin-only and record who set the + cap and when. + + - **Operator-added (overlay) teammates carry no manifest cap**, because they + have no `[[agent]]` entry — but since #343 they can be capped through a + console override like anyone else, including at creation. + + [`CompanyRecord::effective_budget`]: ../../../src/ports/types.rs +- **`[brain]`** selects the `Brain` implementation. `hosted` requires a + TinyHumans credential at runtime; `sidecar` requires the `sidecar` feature. +- **`[[harness]]`** declares the company's named execution engines; the full + story is [harnesses.md](harnesses.md). Each entry has a unique `id` and a + `kind` (`built_in` | `acp`), and exactly one sets `default = true`. An agent + binds with `harness = ""`, or takes the default. **Absent entirely** — the + case for every bundle under `companies/` — means one implicit `built_in` + harness on the company-level `[inference]`, so the table is purely additive. A + section on the wrong kind (`[harness.inference]` on an `acp` entry, or + `[harness.acp]` on a `built_in` one) is a validation error, not an ignored + key. +- **`[inference]`** (issue #56 — BYOK) routes agents through a chosen model + provider, and is the fallback for a harness declaring no + `[harness.inference]`. `provider` is one of `openrouter` (the default) / + `openai_compatible` / `ollama`; `base_url` is required for the latter two. + `api_key_secret` names a **secret-store key** — never the token, which is + written write-only through the console (Connections → Inference). + `[inference.models]` maps an abstract cognition tier (`chat-v1`, + `reasoning-v1`, `agentic-v1`, `vision-v1`) to a concrete OpenRouter model id. + `openrouter` is **dual-mode** — keyless rides the subscription, a tenant + `sk-or-…` goes direct — and the removed `managed` kind aliases to it. Full + detail, including the per-harness secret slots, is in + [providers.md](providers.md). + Precedence is **runtime console override > manifest > platform + default**, and a per-tenant provider re-resolves it every turn — so a console + switch takes effect on the agents' next turn with **no restart**. + That holds only once the company is already on the harness cognition path. + *Which brain a company runs* is decided once, when the runtime is built: a + company that resolved no inference source at boot gets the offline echo brain, + and a credential saved afterwards does not reach it. The status route reports + that as `restartRequired` (issue #266). Since issue #290 the save **rebuilds + the runtime in place** rather than asking for a restart a hosted operator has + no way to perform — see [runtime rebuild](rebuild.md). `restartRequired` stays + honest: still `true` on a host that wired no rebuilder. + Saving the platform default from the console is a *revert* + (`DELETE …/inference`) and + carries no credential, so the console refuses that save while a key is still + typed in the form rather than dropping it and reporting success (issue #265). +- **`[channels.*]`** enables `ChannelAdapter`s. Unknown channels are a + validation error; disabled OpenHuman means non-operator channels degrade + with a boot warning, never a failure. +- **`[policy]`** configures the default `ApprovalGate`. `mode` takes three of + its four names from OpenHuman's security tiers; `auto` is opencompany's own + and sits between `supervised` and `full` (the agent's sandbox writes and + outward reads run unattended, anything that leaves the company or spends on + submit still parks). `always_approve` lists effect kinds that park for + approval regardless of amount and wins over every tier including `full`; + `auto_approve_under_usd` lets small spends through; `approval_ttl_hours` sets + how long a parked approval waits before it default-denies (24 hours by + default — see [approvals.md](../company-brain/approvals.md), issue #971). + Omitting it is not the same as writing `24`: the key stays absent from the + persisted seed, which is what keeps a future change to the default from + looking like an edit and discarding a console `[policy]` override. The parse default is + `supervised`, with all money/publish/filing effects gated — but a **new** + company is given `auto`, written into its manifest explicitly rather than + left to that default. See + [grants.md](../company-brain/grants.md#which-tier-a-new-company-gets) + for why those are two separate knobs, and why moving the parse default is the + one thing issue #605 declined to do. **A tool name is an + effect kind** — the harness projects one onto the other — so + `["publish_artifact"]` and `["payment.send"]` are the same syntax at + different segment counts (issue #684). Operator-authored effect kinds remain + open-ended because a hosted brain may emit a kind this repository has never + seen; the shared matcher runs before the checkpoint taxonomy. The default is + **empty**: `supervised` already parks every money / publish / filing effect + through that taxonomy, so the conservative default is the mode, not the + list. +- **`[place]`** drives the [going-public flow](../company-as-agent/README.md). + `skills` feed Agent Card generation; prices are decimal strings (USDC). +- **`[budget].monthly_usd`** is a hard ceiling enforced by the kernel across + inference usage and x402 spend; reaching it pauses the company with an + operator notification rather than silently degrading. +- **`[plan]`** (issue #108) gates the exec tool families (`shell`, `code`, + `web`, `subagent`) by the company's **token spend this period**, a distinct + axis from `[policy].mode` (autonomy) and an agent's `tier` (cognition). A + built-in `name` (`free` / `starter` / `pro` / `unlimited`) supplies a base + budget map; `token_budgets` overrides/extends it per namespace. The map's key + set **is** the capability set — a gateable namespace absent from it is denied + outright. Each budget is a **threshold over total period token spend**, not a + per-namespace meter (usage samples carry no per-tool attribution): when spend + reaches a tier's budget, that tier's tools switch off for the rest of the + period; different budgets give **graduated degradation**. `period` is `daily` + (default) or `monthly`, aligned to UTC calendar boundaries. Gating is + **fail-closed** — if the usage meter can't be read, every gateable family is + denied (the turn still runs on its intrinsic memory/file/MCP tools). The gate + re-resolves before every turn, so a tier that crosses its budget mid-session + switches off on the **next** turn (a turn already in flight finishes). An + absent `[plan]` leaves gating off entirely. The console's Usage view shows a + live per-tier budget card (`GET …/capabilities`). + - **`media`** (issue #109) is a fifth gateable namespace covering the + image/video generation tools (`media_generate_image`, + `media_generate_video`, `media_list_models`), but it is **real-money and + opt-in**: it is granted only by an **explicit** `media` / `media.*` entry in + `[tools].allow` — the `*` wildcard deliberately does **not** grant it — and + it runs exclusively on a **platform credential** (resolved from the + environment, never a tenant BYOK key or secret). It is absent from the + `free` / `starter` / `pro` tiers (denied there) and uncapped only under + `unlimited`; a company opts in per-namespace with + `token_budgets = { media = N }`. Every generation additionally **parks for + operator approval** before the backend bills it, and the whole family is + compiled out unless the build enables the `media` feature. With no platform + credential configured, a `media` grant wires no tools (fail-closed). The + Usage view surfaces a dedicated media status row (active / awaiting + credential / not granted / not in this build). + - **`search`** (issue #238) is a seventh gateable namespace covering the single + `web_search` tool — source *discovery* for the research skills, which + previously ran on a belt that could read a known URL but never find one. It + is **priced and opt-in**: granted only by an **explicit** `search` / + `search.*` entry in `[tools].allow` (the `*` wildcard deliberately does + **not** grant it, and unlike `media`/`composio` it is **not** in the default + grant list either), and it runs exclusively on the **platform + credential** — the same identity as keyless `openrouter`, resolved from the + environment, never a tenant key. The backend charges per request and reports + the amount, which is recorded as one `SearchCall` usage sample and rolls into + the window's cost. + Three things differ from `media` on purpose: + - **Individual searches do not park for approval.** Consent is the explicit + grant; the boundary is `[tools].search_daily_calls`, a per-company **daily + call cap** (default 200; `0` pauses search without editing `allow`). + Over-cap returns a loud "search budget exhausted" tool error, never an + empty result set — an agent handed silence invents citations. An operator + who does want a per-call gate sets + `[policy].always_approve = ["web_search"]`, which overrides every tier. + - **`[policy].mode = "readonly"` still denies it.** A search reaches a third + party and spends money, so a desk whose contract is that nothing is spent + does not get one. + - **There is no `search` Cargo feature.** The tool rides the `openhuman` + harness feature so CI's gated lane actually compiles and tests it. + + **`max_delegation_depth`** (issue #176) bounds how deep one operator + message's hand-off chain may run, counted in hand-offs: the orchestrator + handing work to a desk lead is level 1, that lead handing a slice on is + level 2. Default `2`; valid `1..=4`, where `1` is the "recursion off" setting + and reproduces the pre-#176 behaviour exactly. + + The depth in force is read from the **live company record** on every call, so + lowering it takes effect on the next turn without a rebuild. A hand-off past + the bound is refused in the model's own turn with the reason + `depth_capped` — while `spawn_task` still works at the bound, so a member that + has run out of chain leaves the remaining work tracked instead of doing it + silently. + + The bound only ever matters to an agent some manifest opted in with + `delegates_to`; a company that names nobody is unaffected by any value here. + It is deliberately low: the fan-out cap applies per level, so each extra + level multiplies the turns one message can buy. + The Usage view surfaces a `Web searches` KPI plus a search status row + (active / paused at cap 0 / awaiting credential / not granted / not in this + build). + - **`workspace`** (issues #237, #551, #671) grants the company's shared note tree — + the `Standards/` / `Playbooks/` / `Product/` documents seeded from + `companies//workspace/**`, plus whatever the operator and the agents + have written since. It is **split**, unlike every other namespace: *reads* + (`workspace_list`, `workspace_search`, `workspace_read`) follow the + ordinary rule, so a + catch-all `*` confers them; *mutations* (`workspace_write`, + `workspace_create`, `workspace_rename`, `workspace_delete`) need an + **explicit** `workspace` or `workspace.write` + entry in `[tools].allow`, because they change a tree every other agent then + treats as the company's source of truth. `workspace.read` is therefore a + genuinely read-only grant. All four ride the one flag on purpose: + overwriting an existing standard is strictly more destructive than adding a + note beside it, and strictly more destructive than removing or moving + something inside the agent's own folder, so a grant permitting the first has + already permitted the rest. Issue #671 deliberately added no fifth grant + name. `workspace_search` (issue #607) is a read and rides the read side of + that split — **not** the metered `search` namespace, despite the name. + `search` is the paid external-credential grant that carries `web_search`; + reading the company's own notes must not require a billed credential, and + search reads exactly what `workspace_read` already grants, so it costs the + operator no additional decision. `workspace_write` overwrites one + **existing** note and requires an + `expected_updated_at` revision token taken from a prior read, so a note + edited in the console since the agent read it is refused rather than + clobbered. `workspace_create` adds one folder or note at a path that is + **free** and whose parent folder already exists — never an overwrite, never + a `mkdir -p`. The single exception is the agent's own + `Agents//`, which is created on demand when the agent writes + directly into it, because that folder is minted on first use rather than + provisioned at boot. Since issue #552 this call is one of two paths that + bring it into existence; publishing a deliverable + (`artifact_mirror::materialize`) is the other, and both go through the same + `ensure_agent_folder` seam. + + `workspace_rename` and `workspace_delete` (issue #671) are the tidying half. + Both act on **one node at a time** and both reach only + `Agents//` — the agent's own folder, never the folder itself, + never a teammate's, never shared guidance. `workspace_delete` carries the + same required `expected_updated_at` token as `workspace_write` and refuses a + folder that still holds anything, so a subtree is removed as N deliberate, + individually-parked calls rather than one. `workspace_rename` carries no + token, because it destroys nothing: body, id and both authorship stamps + survive it. Read the confinement as a division of labour rather than as a + security boundary — the same grant already confers *unconfined* overwrite, + which reaches further than own-folder lifecycle does. Renaming or deleting + anything elsewhere in the tree stays operator-only. + + Agent writes are broad: an agent may create or edit ordinary shared content + anywhere in its company's tree. The reserved lowercase `secrets/` subtree + is the exception: boot creates it with an explanatory `README.md`, and + agent workspace list/read/search/write/create tools omit or refuse the + entire subtree while operator workspace APIs retain full access. This is a + model-visibility boundary, not the application credential store; provider + and tool credentials still belong in Connections/inference settings. + Confining other creation while leaving overwrite free would + protect nothing. What keeps the tree navigable instead is steering plus + attribution — the persona brief names the agent's own reserved folder + `Agents//` (minted the first time that agent puts something in it; + boot scaffolds the empty `Agents/` root plus `secrets/README.md`, and since + issue #645 `Desks/` is minted on first use rather than scaffolded) as the default + home for what it produces and marks shared + guidance as something to edit only on purpose, and every node records who + created it and who last wrote it (issue #326), which the console shows. Both + sides are capped at the agent harness's own per-tool-result byte budget + minus the framing a read wraps a body in (issue #417) — 12 KiB today, + derived rather than chosen so a full read always survives the harness cut + and the write gate is measured against the bytes the model actually + received. A larger note is agent-read-only: the agent sees a truncated body, + and a write against it is refused rather than discarding the part it could + not see, so only an operator can edit it in the console. **Operator edits + are not capped by this** — the console and the REST handlers write through + the `WorkspaceStore` port directly and never enter the agent tool path. Under + `[policy].mode = "supervised"` (the default) a write additionally parks for + approval, and under `readonly` it is denied — reads stay available in every + mode. The namespace is **not** gateable by `[plan].token_budgets`: reads + cost nothing and shedding them would only make agents guess at company + standards. The tools hit the store per call, so an operator's console edit + is visible to the next turn with no restart. +- **`[[schedule]]`** entries become `ScheduleFired` events; cron syntax is + standard 5-field, interpreted in UTC. A saved *workflow* schedules itself + separately, with the same dialect: its `trigger` node carries a `schedule` + cron that the workflow scheduler fires (issue #169). A manifest schedule + drives a company cycle; a trigger schedule drives one workflow run. +- **`[workflows]`** enables saved graphs and bounds how many may run at once. + `enabled` lists the `workflows/.toml` ids to turn on. `max_in_flight_runs` + (issue #401) is the company's concurrent-run ceiling — default **8**, + validated **>= 1** (a `0` would refuse every run and is rejected at load). It + applies to *every* entry point that starts a run (the manual run route, the + cron scheduler, an approved gate's continuation, and the orchestrator's + `run_workflow` tool), enforced at one choke point. The default sits above 1 + deliberately: a running workflow's agent node can call `run_workflow` while + the parent run still holds a slot, so a ceiling of 1 would refuse legitimate + nesting. A run over the ceiling is **refused, never queued** — the run route + answers `429` (see `api.md`), a scheduled fire is skipped for that minute, and + the orchestrator tool tells the agent to wait. A slot frees the instant a run + settles. diff --git a/docs/spec/runtime/manifest.md b/docs/spec/runtime/manifest.md index 8950cd961..6581a5f8a 100644 --- a/docs/spec/runtime/manifest.md +++ b/docs/spec/runtime/manifest.md @@ -64,12 +64,28 @@ wallets = ["7xKXtg2CW87d97TXJSDpbD5jBkheTqA83TZRuJosgAsU"] mode = "hosted" # hosted (default) | sidecar max_passes = 12 # passed through to Medulla -[inference] # NEW: per-tenant Bring-Your-Own-Key (#56) -provider = "openrouter" # managed (default) | openrouter | openai_compatible | ollama +[[harness]] # named execution engines — see harnesses.md +id = "embedded" # snake_case, unique +kind = "built_in" # built_in (default) | acp +default = true # exactly one entry, when any is declared + +[harness.inference] # attaches to the [[harness]] above +provider = "openrouter" + +# [[harness]] +# id = "my_laptop" +# kind = "acp" +# [harness.acp] +# transport = "local" # local | runner +# agent = "claude" # claude | codex | goose + +[inference] # per-tenant BYOK (#56) — the fallback for a +provider = "openrouter" # harness declaring no [harness.inference]. + # openrouter (default) | openai_compatible | ollama # base_url = "https://openrouter.ai/api/v1" # required for ollama/openai_compatible; defaulted otherwise # api_key_secret = "byo/openrouter" # names a secret-store KEY — never the token itself -[inference.models] # abstract tier → concrete provider model id +[inference.models] # abstract tier → concrete OpenRouter model id "chat-v1" = "deepseek/deepseek-chat" "reasoning-v1" = "deepseek/deepseek-r1" @@ -127,355 +143,12 @@ cron = "0 9 * * MON" prompt = "Weekly review and operator digest" ``` -## Semantics - -- **`[company]`** becomes the seed of the [Charter](../company-brain/charter.md). - `handle` is only used when `[place].discoverable = true`. -- **`[[agent]]`** entries define the Roster — or, equivalently, one - `agents/.toml` file per teammate under the company bundle, carrying the - same keys with the filename as the id. The two forms are **exclusive**: - declaring both is a validation error rather than a precedence rule, because - either precedence would silently discard teammates somebody wrote down. Full - schema, including `prompt` / `prompt_files` / `context` / `classes`, in - [runtime/agents.md](agents.md). - - `tier` is a hint the brain may use when delegating; it never selects a model - (the backend maps tiers to SKUs). `tools` and `budget_usd_daily` intersect - with the company-wide `[tools].allow` and `[budget]` — the most restrictive - wins. Tool grants resolve through **three** levels, - `[tools].allow ∩ [[group_chat]].tools ∩ [[agent]].tools`, every one of them - narrow-only and an empty one a pass-through; see - [runtime/tools.md](tools.md), which also covers why an empty grant list means - "inherit" rather than "nothing". - - **`delegates_to`** (issue #176) is the one per-agent key that is *not* a - narrowing of a company-wide list: it is an **opt-in**. Empty — the default, - and every manifest written before it existed — means the agent carries no - delegation tool at all, which is how a dispatched desk agent has always - behaved. Naming one or more desks wires exactly two tools onto it, - `spawn_task` and a `delegate_to_desk` narrowed to those desks, so a desk lead - can pull a specialist in for one slice instead of handing the whole request - back to the orchestrator. - - It takes **desk** ids or names (`[[group_chat]]` entries), never teammate - ids — desks are the address space `delegate_to_desk` already resolves - against — and `"*"` means every desk the company has. An entry that names no - declared desk fails validation, because at runtime it would fail silently: - the member would carry the tool and every call would be refused. - - It never confers the orchestrator's *authority* — `assign_task`, - `review_task`, `add_agent`, `query_company`, `run_workflow`, - `create_workflow`, and the #661 workflow-admin trio (`read_workflow`, - `update_workflow`, `delete_workflow`) stay orchestrator-only. A member gets what it needs to pass - a slice on and to leave the rest tracked, and nothing more. - - Three runtime guards bound what it can do, all enforced at the tool boundary - in the member's own turn rather than by which tools were wired (belts are - cached per roster, so a tool cannot be withheld from one turn): - - - **Depth** — `[tools].max_delegation_depth`, below. - - **Cycles** — a hand-off to a desk already on the current chain (A→B→A), or - to the desk the caller itself leads, is refused. - - **Allowlist** — a target outside `delegates_to` is refused, and the refusal - names the desks the member *can* reach so it can retry in the same turn. - - Each refusal reaches both the model and the board: the run trail carries it - verbatim, and a refused hand-off is recorded on the dispatched card's note, - so the operator reads the fact rather than inferring it from an absence. - - The per-turn fan-out cap (three delegations) applies **per level**, not per - message — each turn starts against an empty queue. - - **`budget_usd_daily`** (enforced since issue #304 — before that it was - validated, stored and displayed, but nothing read it) caps one teammate's - spend over the **UTC calendar day**, resetting at `00:00Z` — the same - boundary `[plan]`'s daily token budget uses. Spend is re-read from the usage - meter on every check, so a restart mid-day resumes against the real figure - rather than a fresh zero. - It covers **metered, attributed** spend: inference turns and priced tool - calls (`web_search`, `media_generate_*`), plus any tool call that declares an - `amount_usd`. Two behaviours at the cap: - - - **Dispatch is refused** for that teammate, before any model call, with a - notice naming the cap and the reset. The rest of the company keeps running. - - **A priced tool call parks for approval** rather than being denied. - Approving it runs that one call and nothing more; the cap is not raised. - Free reads and sends are unaffected — a spend cap caps spend. - - Known limits, stated rather than papered over: - - - **Executed x402 payments escape the counter.** Ledger entries carry no - agent, so there is nothing to attribute them to. What *is* covered is the - pre-flight case: a call declaring an `amount_usd` that would breach the - remaining budget parks before the money moves. Company-wide payment - spending is governed by `[budget].monthly_usd`, which is enforced on the - economy path. - - **Turn-boundary overshoot.** A turn or call that starts under the cap can - finish over it; the overshoot is bounded by one call. There is no - reservation ledger in v1 — the same documented window `[plan]` carries. - - **Unreadable spend fails differently at each layer, deliberately.** With no - meter or a failing meter, dispatch **runs** (bricking a teammate's - cognition with no operator recourse is worse than a day of overspend), - while a priced tool call **parks** (a human can wave that one through). - Since issue #343 the manifest value is a **default, not the last word**. An - admin can set, change or clear a teammate's cap from the console - (`PUT`/`DELETE …/team/{agentId}/budget`); the override is stored on the - company record, wins over `budget_usd_daily` everywhere the cap is read - ([`CompanyRecord::effective_budget`] is the single reconciliation point), and - takes effect on the teammate's **next dispatch** — no restart, no redeploy. - That matters most on a hosted tenant, where `company.toml` is baked into the - container image and an operator has no way to edit it. Three stored states, - kept deliberately distinct: no override (the manifest applies), a cap of `x` - (`0` included, meaning "may not spend"), and an explicit "uncapped" that beats - a manifest cap. `DELETE` drops the override so the manifest applies again, - which no `PUT` body can express. Writes are admin-only and record who set the - cap and when. - - - **Operator-added (overlay) teammates carry no manifest cap**, because they - have no `[[agent]]` entry — but since #343 they can be capped through a - console override like anyone else, including at creation. - - [`CompanyRecord::effective_budget`]: ../../../src/ports/types.rs -- **`[brain]`** selects the `Brain` implementation. `hosted` requires a - TinyHumans credential at runtime; `sidecar` requires the `sidecar` feature. -- **`[inference]`** (issue #56 — BYOK) routes the company's agents through a - chosen model provider. Absent (the default) keeps the managed TinyHumans - brain. `provider` is one of `managed` / `openrouter` / `openai_compatible` / - `ollama`; `base_url` is required for the latter two. `api_key_secret` names a - **secret-store key** — never the token, which is written write-only through - the console (Connections → Inference). `[inference.models]` maps an abstract - cognition tier (`chat-v1`, `reasoning-v1`, `agentic-v1`, `vision-v1`) to a - concrete provider model id; an unmapped tier passes through verbatim. - Precedence is **runtime console override > manifest `[inference]` > managed - default**, and a per-tenant provider re-resolves it every turn — so a console - switch takes effect on the agents' next turn with **no restart**. - That holds only once the company is already on the harness cognition path. - *Which brain a company runs* is decided once, when the runtime is built: a - company that resolved no inference source at boot gets the offline echo brain - and an unwired workflow runner, and a credential saved afterwards reaches - neither. The status route reports that state as `restartRequired` — a resolved - config next to a non-harness `cognition` — and the console says "restart" - instead of "next turn" for it (issue #266). - Since issue #290 that save **rebuilds the company's runtime in place** rather - than asking for a restart the operator may have no way to perform: a hosted - tenant's unit of restart is its container, and the control plane has no button - for it. `PUT …/inference` quiesces the running runtime, hands its live state to - a successor, and swaps the registry — see - [runtime rebuild](rebuild.md). `restartRequired` remains on the read shape and - stays honest: it is still `true` on a host that wired no rebuilder, which is - when a restart genuinely is the only route. - Saving `managed` from the console is a *revert* (`DELETE …/inference`) and - carries no credential, so the console refuses that save while a key is still - typed in the form rather than dropping it and reporting success (issue #265). -- **`[channels.*]`** enables `ChannelAdapter`s. Unknown channels are a - validation error; disabled OpenHuman means non-operator channels degrade - with a boot warning, never a failure. -- **`[policy]`** configures the default `ApprovalGate`. `mode` takes three of - its four names from OpenHuman's security tiers; `auto` is opencompany's own - and sits between `supervised` and `full` (the agent's sandbox writes and - outward reads run unattended, anything that leaves the company or spends on - submit still parks). `always_approve` lists effect kinds that park for - approval regardless of amount and wins over every tier including `full`; - `auto_approve_under_usd` lets small spends through; `approval_ttl_hours` sets - how long a parked approval waits before it default-denies (24 hours by - default — see [approvals.md](../company-brain/approvals.md), issue #971). - Omitting it is not the same as writing `24`: the key stays absent from the - persisted seed, which is what keeps a future change to the default from - looking like an edit and discarding a console `[policy]` override. The parse default is - `supervised`, with all money/publish/filing effects gated — but a **new** - company is given `auto`, written into its manifest explicitly rather than - left to that default. See - [grants.md](../company-brain/grants.md#which-tier-a-new-company-gets) - for why those are two separate knobs, and why moving the parse default is the - one thing issue #605 declined to do. **A tool name is an - effect kind** — the harness projects one onto the other — so - `["publish_artifact"]` and `["payment.send"]` are the same syntax at - different segment counts (issue #684). Operator-authored effect kinds remain - open-ended because a hosted brain may emit a kind this repository has never - seen; the shared matcher runs before the checkpoint taxonomy. The default is - **empty**: `supervised` already parks every money / publish / filing effect - through that taxonomy, so the conservative default is the mode, not the - list. -- **`[place]`** drives the [going-public flow](../company-as-agent/README.md). - `skills` feed Agent Card generation; prices are decimal strings (USDC). -- **`[budget].monthly_usd`** is a hard ceiling enforced by the kernel across - inference usage and x402 spend; reaching it pauses the company with an - operator notification rather than silently degrading. -- **`[plan]`** (issue #108) gates the exec tool families (`shell`, `code`, - `web`, `subagent`) by the company's **token spend this period**, a distinct - axis from `[policy].mode` (autonomy) and an agent's `tier` (cognition). A - built-in `name` (`free` / `starter` / `pro` / `unlimited`) supplies a base - budget map; `token_budgets` overrides/extends it per namespace. The map's key - set **is** the capability set — a gateable namespace absent from it is denied - outright. Each budget is a **threshold over total period token spend**, not a - per-namespace meter (usage samples carry no per-tool attribution): when spend - reaches a tier's budget, that tier's tools switch off for the rest of the - period; different budgets give **graduated degradation**. `period` is `daily` - (default) or `monthly`, aligned to UTC calendar boundaries. Gating is - **fail-closed** — if the usage meter can't be read, every gateable family is - denied (the turn still runs on its intrinsic memory/file/MCP tools). The gate - re-resolves before every turn, so a tier that crosses its budget mid-session - switches off on the **next** turn (a turn already in flight finishes). An - absent `[plan]` leaves gating off entirely. The console's Usage view shows a - live per-tier budget card (`GET …/capabilities`). - - **`media`** (issue #109) is a fifth gateable namespace covering the - image/video generation tools (`media_generate_image`, - `media_generate_video`, `media_list_models`), but it is **real-money and - opt-in**: it is granted only by an **explicit** `media` / `media.*` entry in - `[tools].allow` — the `*` wildcard deliberately does **not** grant it — and - it runs exclusively on a **managed platform credential** (resolved from the - environment, never a tenant BYOK key or secret). It is absent from the - `free` / `starter` / `pro` tiers (denied there) and uncapped only under - `unlimited`; a company opts in per-namespace with - `token_budgets = { media = N }`. Every generation additionally **parks for - operator approval** before the backend bills it, and the whole family is - compiled out unless the build enables the `media` feature. With no managed - credential configured, a `media` grant wires no tools (fail-closed). The - Usage view surfaces a dedicated media status row (active / awaiting - credential / not granted / not in this build). - - **`search`** (issue #238) is a seventh gateable namespace covering the single - `web_search` tool — source *discovery* for the research skills, which - previously ran on a belt that could read a known URL but never find one. It - is **priced and opt-in**: granted only by an **explicit** `search` / - `search.*` entry in `[tools].allow` (the `*` wildcard deliberately does - **not** grant it, and unlike `media`/`composio` it is **not** in the default - grant list either), and it runs exclusively on the **managed platform - credential** — the same identity as managed inference, resolved from the - environment, never a tenant key. The backend charges per request and reports - the amount, which is recorded as one `SearchCall` usage sample and rolls into - the window's cost. - Three things differ from `media` on purpose: - - **Individual searches do not park for approval.** Consent is the explicit - grant; the boundary is `[tools].search_daily_calls`, a per-company **daily - call cap** (default 200; `0` pauses search without editing `allow`). - Over-cap returns a loud "search budget exhausted" tool error, never an - empty result set — an agent handed silence invents citations. An operator - who does want a per-call gate sets - `[policy].always_approve = ["web_search"]`, which overrides every tier. - - **`[policy].mode = "readonly"` still denies it.** A search reaches a third - party and spends money, so a desk whose contract is that nothing is spent - does not get one. - - **There is no `search` Cargo feature.** The tool rides the `openhuman` - harness feature so CI's gated lane actually compiles and tests it. - - **`max_delegation_depth`** (issue #176) bounds how deep one operator - message's hand-off chain may run, counted in hand-offs: the orchestrator - handing work to a desk lead is level 1, that lead handing a slice on is - level 2. Default `2`; valid `1..=4`, where `1` is the "recursion off" setting - and reproduces the pre-#176 behaviour exactly. - - The depth in force is read from the **live company record** on every call, so - lowering it takes effect on the next turn without a rebuild. A hand-off past - the bound is refused in the model's own turn with the reason - `depth_capped` — while `spawn_task` still works at the bound, so a member that - has run out of chain leaves the remaining work tracked instead of doing it - silently. - - The bound only ever matters to an agent some manifest opted in with - `delegates_to`; a company that names nobody is unaffected by any value here. - It is deliberately low: the fan-out cap applies per level, so each extra - level multiplies the turns one message can buy. - The Usage view surfaces a `Web searches` KPI plus a search status row - (active / paused at cap 0 / awaiting credential / not granted / not in this - build). - - **`workspace`** (issues #237, #551, #671) grants the company's shared note tree — - the `Standards/` / `Playbooks/` / `Product/` documents seeded from - `companies//workspace/**`, plus whatever the operator and the agents - have written since. It is **split**, unlike every other namespace: *reads* - (`workspace_list`, `workspace_search`, `workspace_read`) follow the - ordinary rule, so a - catch-all `*` confers them; *mutations* (`workspace_write`, - `workspace_create`, `workspace_rename`, `workspace_delete`) need an - **explicit** `workspace` or `workspace.write` - entry in `[tools].allow`, because they change a tree every other agent then - treats as the company's source of truth. `workspace.read` is therefore a - genuinely read-only grant. All four ride the one flag on purpose: - overwriting an existing standard is strictly more destructive than adding a - note beside it, and strictly more destructive than removing or moving - something inside the agent's own folder, so a grant permitting the first has - already permitted the rest. Issue #671 deliberately added no fifth grant - name. `workspace_search` (issue #607) is a read and rides the read side of - that split — **not** the metered `search` namespace, despite the name. - `search` is the paid external-credential grant that carries `web_search`; - reading the company's own notes must not require a billed credential, and - search reads exactly what `workspace_read` already grants, so it costs the - operator no additional decision. `workspace_write` overwrites one - **existing** note and requires an - `expected_updated_at` revision token taken from a prior read, so a note - edited in the console since the agent read it is refused rather than - clobbered. `workspace_create` adds one folder or note at a path that is - **free** and whose parent folder already exists — never an overwrite, never - a `mkdir -p`. The single exception is the agent's own - `Agents//`, which is created on demand when the agent writes - directly into it, because that folder is minted on first use rather than - provisioned at boot. Since issue #552 this call is one of two paths that - bring it into existence; publishing a deliverable - (`artifact_mirror::materialize`) is the other, and both go through the same - `ensure_agent_folder` seam. - - `workspace_rename` and `workspace_delete` (issue #671) are the tidying half. - Both act on **one node at a time** and both reach only - `Agents//` — the agent's own folder, never the folder itself, - never a teammate's, never shared guidance. `workspace_delete` carries the - same required `expected_updated_at` token as `workspace_write` and refuses a - folder that still holds anything, so a subtree is removed as N deliberate, - individually-parked calls rather than one. `workspace_rename` carries no - token, because it destroys nothing: body, id and both authorship stamps - survive it. Read the confinement as a division of labour rather than as a - security boundary — the same grant already confers *unconfined* overwrite, - which reaches further than own-folder lifecycle does. Renaming or deleting - anything elsewhere in the tree stays operator-only. +## Semantics - Agent writes are broad: an agent may create or edit ordinary shared content - anywhere in its company's tree. The reserved lowercase `secrets/` subtree - is the exception: boot creates it with an explanatory `README.md`, and - agent workspace list/read/search/write/create tools omit or refuse the - entire subtree while operator workspace APIs retain full access. This is a - model-visibility boundary, not the application credential store; provider - and tool credentials still belong in Connections/inference settings. - Confining other creation while leaving overwrite free would - protect nothing. What keeps the tree navigable instead is steering plus - attribution — the persona brief names the agent's own reserved folder - `Agents//` (minted the first time that agent puts something in it; - boot scaffolds the empty `Agents/` root plus `secrets/README.md`, and since - issue #645 `Desks/` is minted on first use rather than scaffolded) as the default - home for what it produces and marks shared - guidance as something to edit only on purpose, and every node records who - created it and who last wrote it (issue #326), which the console shows. Both - sides are capped at the agent harness's own per-tool-result byte budget - minus the framing a read wraps a body in (issue #417) — 12 KiB today, - derived rather than chosen so a full read always survives the harness cut - and the write gate is measured against the bytes the model actually - received. A larger note is agent-read-only: the agent sees a truncated body, - and a write against it is refused rather than discarding the part it could - not see, so only an operator can edit it in the console. **Operator edits - are not capped by this** — the console and the REST handlers write through - the `WorkspaceStore` port directly and never enter the agent tool path. Under - `[policy].mode = "supervised"` (the default) a write additionally parks for - approval, and under `readonly` it is denied — reads stay available in every - mode. The namespace is **not** gateable by `[plan].token_budgets`: reads - cost nothing and shedding them would only make agents guess at company - standards. The tools hit the store per call, so an operator's console edit - is visible to the next turn with no restart. -- **`[[schedule]]`** entries become `ScheduleFired` events; cron syntax is - standard 5-field, interpreted in UTC. A saved *workflow* schedules itself - separately, with the same dialect: its `trigger` node carries a `schedule` - cron that the workflow scheduler fires (issue #169). A manifest schedule - drives a company cycle; a trigger schedule drives one workflow run. -- **`[workflows]`** enables saved graphs and bounds how many may run at once. - `enabled` lists the `workflows/.toml` ids to turn on. `max_in_flight_runs` - (issue #401) is the company's concurrent-run ceiling — default **8**, - validated **>= 1** (a `0` would refuse every run and is rejected at load). It - applies to *every* entry point that starts a run (the manual run route, the - cron scheduler, an approved gate's continuation, and the orchestrator's - `run_workflow` tool), enforced at one choke point. The default sits above 1 - deliberately: a running workflow's agent node can call `run_workflow` while - the parent run still holds a slot, so a ceiling of 1 would refuse legitimate - nesting. A run over the ceiling is **refused, never queued** — the run route - answers `429` (see `api.md`), a scheduled fire is skipped for that minute, and - the orchestrator tool tells the agent to wait. A slot frees the instant a run - settles. +The behaviour of every key and table in the schema above is spelled out in +[manifest-semantics.md](manifest-semantics.md) — split out so this page stays +under the 500-line cap while the schema stays discoverable here. ## Layering and provenance diff --git a/docs/spec/runtime/providers.md b/docs/spec/runtime/providers.md new file mode 100644 index 000000000..729381c04 --- /dev/null +++ b/docs/spec/runtime/providers.md @@ -0,0 +1,215 @@ +# Inference providers + +*Which models a `built_in` harness reaches, and who pays.* + +Terms: [glossary](../glossary.md). Which harness consults this at all is +[harnesses.md](harnesses.md); credential doctrine is +[credentials.md](credentials.md). + +--- + +## The provider set + +| provider | endpoint | credential | +|---|---|---| +| `openrouter` | dual-mode, below | optional tenant `sk-or-…` | +| `openai_compatible` | required `base_url` | usually a key | +| `ollama` | `base_url`, defaulting to a local server | none | + +`openrouter` is the default. There is no provider for OpenCompany's own models, +because OpenCompany does not host models — the spec non-goal "not a model host" +is load-bearing here, and the provider list is where it shows. + +### `managed` is gone + +`managed` named the hosted TinyHumans brain and addressed proprietary SKUs. +OpenCompany no longer exposes its own models, so there is nothing left for a +distinct kind to name. + +A manifest or stored runtime blob still saying `managed` **aliases** to +`openrouter` rather than failing. It named a real thing when it was written, and +the intent — "the platform's brain" — is exactly what proxied OpenRouter is. +Rejecting it would break bundles that were valid, to no purpose. An *unknown* +provider is a different matter and fails loudly; see below. + +--- + +## `openrouter` is dual-mode + +Which mode a company is in depends only on whether it holds a key: + +| tenant key | endpoint | credential | telemetry slug | +|---|---|---|---| +| **absent** | the platform endpoint | the platform token | `subscription` | +| `sk-or-…` | `https://openrouter.ai/api/v1` | the tenant's key | `openrouter` | + +**Keyless is the default a company starts on**, and it must be a working config +rather than a prompt for a credential: the platform proxy fronts OpenRouter +upstream and meters the spend against the subscription. From the workload's +point of view the two endpoints serve the same catalogue; only who pays differs. + +The keyless branch inherits the platform's base URL *and* credential. Without +that inheritance a company naming a provider but holding no key of its own would +401 instead of riding the subscription — which is why the branch survived +`managed`'s removal rather than being deleted with it. + +`InferenceDecl::is_proxied()` records which mode resolved. + +### Setting a key moves you off the proxy + +A stored key is an *OpenRouter* key, so it goes to OpenRouter. Sending an +`sk-or-…` to the platform proxy would simply be rejected. + +> **Behaviour change.** Under `managed`, a console-set key kept the platform +> endpoint, so an admin could bill their own account through the proxy. That +> combination no longer exists. An admin who wants the platform endpoint with a +> credential of their own names `openai_compatible` with that `base_url`. + +Clearing the key returns the company to the subscription rather than 401ing, +which is what makes a key genuinely optional in both directions. + +--- + +## Per-harness configuration + +Each `built_in` harness owns its provider, credential slot and model map, so two +harnesses on one company can hold two different OpenRouter accounts. A harness +declaring no `[harness.inference]` falls back to the company-level +`[inference]`. + +### Secret slots + +```text +inference/config # the DEFAULT harness +inference/key + +harness//inference/config # every other harness +harness//inference/key +``` + +The default harness keeps the **flat legacy keys**. This asymmetry is +deliberate: a tenant's stored console override and credential already live at +the flat paths, and the `SecretStore` port has no rename +([ports.md](ports.md)) — so namespacing every harness would silently orphan the +configuration of every company already running. + +### Precedence, within a harness + +1. **Runtime** — what the console wrote, in that harness's `…/config` slot. +2. **Manifest** — its `[harness.inference]`, else the company's `[inference]`. +3. **Default** — the platform-injected endpoint and token. + +Unchanged from the single-provider design; what differs per harness is only +*which* slots tiers 1 and 2 read. + +The credential is resolved **per request**, not captured at boot, because a +hosted tenant's platform credential is a projected token the platform rotates in +place. A value captured once would go stale within minutes. The same deferral is +what makes a console key rotation reach agents on their next turn with no +restart. + +### Credentials are never inline + +`api_key_secret` names a `SecretStore` key. It is never the token. Validation +rejects a value that looks like a pasted credential, so a secret cannot land in +a committed manifest. `InferenceDecl` derives no `Serialize` and its `Debug` +redacts the credential; no read route returns it, and the console sees only a +`keyConfigured` boolean. + +This slot is **not** the company's TinyHumans identity — that is +`tinyhumans/key`, and the distinction is spelled out in +[credentials.md](credentials.md). This one is provider-scoped: whatever the +declared provider wants. Handing it to the TinyHumans backend would present one +vendor's credential to another. + +--- + +## Models + +Agents address workloads by abstract **tier** — `chat-v1`, `reasoning-v1`, +`agentic-v1`, `vision-v1` — derived from the agent's `tier` field. A tier names +a workload, never a model, which is what lets an agent keep its tier while +moving between harnesses. + +What goes on the wire differs by path, and sending the wrong one fails +(`inference::model_for_tier`): + +| path | wire value | why | +|---|---|---| +| proxied | the tier name (`chat-v1`) | the platform's registry routes on it, pinning each tier to a sub-provider so its rate card stays exact | +| direct | a concrete slug (`deepseek/deepseek-v4-flash`) | OpenRouter has never heard of `chat-v1` | + +On the direct path an unmapped tier takes `DEFAULT_TIER_MODELS`, which mirrors +the platform's own OpenRouter bindings — so both paths reach the same models and +adding a key does not silently move a company onto different ones. + +A harness's own `models` entry is honoured **verbatim on both paths**: the +operator named a specific model, and rewriting it is not ours to do. + +### Naming a specific model on the proxied path + +The platform endpoint does accept a concrete model, but under its own +`openrouter//` namespace — an explicit prefix, so an arbitrary +caller string can never reach an upstream URL — and only when passthrough is +switched on there, which is **opt-in and off by default**. It prices such a +request from OpenRouter's live catalog and caps upstream spend at that rate. + +So a bare tier is the only value that always works proxied. An operator who +wants a specific model through the proxy writes the `openrouter/…` form into +`models` themselves, and it is forwarded untouched. + +--- + +## Outbound headers + +| header | when | why | +|---|---|---| +| `HTTP-Referer`, `X-Title` | every `openrouter` request | OpenRouter's own dashboard and rankings | +| `x-sdk-name` | **proxied only** | our endpoint, our telemetry | + +The product-identity header is keyed on `is_proxied()`, not on the provider +kind. After `managed`'s removal the kind no longer distinguishes our endpoint +from OpenRouter's — the same `openrouter` kind reaches both — and it is the +endpoint, not the vocabulary, that this rule is about. + +It must never reach a third party. A tenant's own OpenRouter account, a +self-hosted OpenAI-compatible server and a local Ollama all belong to operators +who have no relationship with TinyHumans and gain nothing from learning which +product a tenant runs. + +--- + +## Unknown providers fail loudly + +An unrecognised kind is an error at resolution, not a fallback. The manifest +validator already rejects one, but a **stored runtime blob never passes through +it** — the console wrote it, possibly under an older build whose vocabulary +differed. Resolving one silently would attribute its spend to whatever the +fallback happened to be, hiding the misconfiguration behind a plausible bill. + +For the same reason `provider_slug` reports `unknown` rather than folding an +unrecognised kind into a real provider's attribution. + +--- + +## Telemetry + +Usage samples carry the slug of the config that actually served the turn, read +live after each turn rather than baked at build — so a console key switch +re-attributes spend on the *next* turn. With named harnesses this is per agent, +so a Usage view separates what each harness spent. + +`subscription` and `openrouter` stay distinct slugs because they are two +different payers, and merging them would tell the operator nothing. + +--- + +## Implementation map + +| concern | where | +|---|---| +| provider vocabulary, defaults | `src/company/types.rs` | +| resolution, scoping, aliasing | `src/company/inference.rs` | +| the chat models and request plan | `src/harness/built_in/provider.rs` | +| read/write plane | `src/server/ops/inference.rs` | +| the subscription proxy itself | the TinyHumans backend | diff --git a/examples/live_company_turn.rs b/examples/live_company_turn.rs index 47a78f844..ba743f510 100644 --- a/examples/live_company_turn.rs +++ b/examples/live_company_turn.rs @@ -106,6 +106,7 @@ async fn main() -> anyhow::Result<()> { ledger_registry: Default::default(), provider: Arc::new(HostedProvider::new(cfg)), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir.path())), store: Arc::new(FsCompanyStore::new(dir.path())), meter: Some(meter.clone()), diff --git a/frontend/test/e2e/inference.spec.ts b/frontend/test/e2e/inference.spec.ts index 57a2f2350..c4c76ca8e 100644 --- a/frontend/test/e2e/inference.spec.ts +++ b/frontend/test/e2e/inference.spec.ts @@ -80,7 +80,7 @@ test("a key typed for a BYOK provider is not discarded by switching to managed", const body = await after.json(); expect(body.keyConfigured).toBe(true); // Setting only a key must not move the company off the managed brain. - expect(body.provider).toBe("managed"); + expect(body.provider).toBe("openrouter"); // And it can be taken back off again — set / rotate / clear, all from here. await page.getByTestId("inference-remove-key").click(); @@ -121,4 +121,16 @@ test("a key typed for a BYOK provider does reach the host on save", async ({ pag await expect(page.getByText("Reverted to the managed configuration.")).toBeVisible({ timeout: 30_000, }); + + // The reset is a full one, not a half-clear: the host also wipes the stored + // credential on revert (issue #993), so nothing is left behind to reroute the + // later specs in this lane (the live-brain workflow and MCP-agent specs) off + // the mock brain and 401 them. Assert that here rather than clearing by hand + // — the remove-key button exists only while a key is stored, so it being gone + // is the observable that the reset actually cleared the key. + await expect(page.getByTestId("inference-remove-key")).toHaveCount(0, { + timeout: 30_000, + }); + const cleared = await page.request.get("/api/v1/company/inference"); + expect((await cleared.json()).keyConfigured).toBe(false); }); diff --git a/scripts/ci/feature-lanes.txt b/scripts/ci/feature-lanes.txt index 536175398..214fd72a1 100644 --- a/scripts/ci/feature-lanes.txt +++ b/scripts/ci/feature-lanes.txt @@ -57,11 +57,11 @@ tinyplace | tested | tinyplace | Runs the whole sqlite | partial | sqlite | store::sqlite mongodb | partial | mongodb | store::mongodb store::select -acp | partial | acp,runner,tinymemory | server::acp harness::acp_run_turn +acp | partial | acp,runner,tinymemory | server::acp harness::acp::run_turn runner | partial | acp,runner,tinymemory | runner -mcp | partial | openhuman,mcp,telegram,media | harness::build::tests app::types -media | partial | openhuman,mcp,telegram,media | harness::build::tests harness::toolbelt -composio | partial | openhuman,tinycortex,chargebee,paypal,composio | harness::composio::isolation_tests harness::composio::ops_helper_tests harness::composio::live::live_tests server::ops::composio::tests::gated_tests +mcp | partial | openhuman,mcp,telegram,media | harness::built_in::build::tests app::types +media | partial | openhuman,mcp,telegram,media | harness::built_in::build::tests harness::built_in::toolbelt +composio | partial | openhuman,tinycortex,chargebee,paypal,composio | harness::built_in::composio::isolation_tests harness::built_in::composio::ops_helper_tests harness::built_in::composio::live::live_tests server::ops::composio::tests::gated_tests export | partial | export | store::export imap | partial | imap,smtp | server::ops::imap chargebee | partial | openhuman,tinycortex,chargebee,paypal,composio | chargebee diff --git a/src/company/agent_file.rs b/src/company/agent_file.rs index 71341ecac..5bd5d46f5 100644 --- a/src/company/agent_file.rs +++ b/src/company/agent_file.rs @@ -81,6 +81,11 @@ struct AgentFile { description: Option, #[serde(default)] tier: Option, + /// Which `[[harness]]` this agent runs on. Cross-checked against the + /// company's declared harnesses in `CompanyManifest::validate`, not here — + /// this file cannot see them. + #[serde(default)] + harness: Option, #[serde(default)] tools: Vec, #[serde(default)] @@ -251,6 +256,7 @@ fn parse_agent_file( role, description: file.description, tier: file.tier, + harness: file.harness, tools: file.tools, delegates_to: file.delegates_to, context: file.context, diff --git a/src/company/context_routing.rs b/src/company/context_routing.rs index 2ae9f3b9b..a151be56e 100644 --- a/src/company/context_routing.rs +++ b/src/company/context_routing.rs @@ -259,6 +259,7 @@ mod tests { name: None, description: None, tier: tier.map(str::to_string), + harness: None, tools: Vec::new(), delegates_to: Vec::new(), context: None, diff --git a/src/company/inference.rs b/src/company/inference.rs index 42cfb1031..09dbb31c3 100644 --- a/src/company/inference.rs +++ b/src/company/inference.rs @@ -44,9 +44,155 @@ pub const RUNTIME_CONFIG_KEY: &str = "inference/config"; /// stored here (write-only via the console); the value is the raw token string. pub const KEY_KEY: &str = "inference/key"; -/// Default managed base URL (the hosted TinyHumans / Medulla OpenAI-compatible -/// surface) when the managed provider names no `base_url`. -pub const MANAGED_BASE_URL: &str = "https://api.tinyhumans.ai/openai/v1"; +/// The [`SecretStore`](crate::ports::SecretStore) key holding the runtime +/// inference override for `harness_id`. +/// +/// The **default** harness keeps the flat legacy key. That is not cosmetic: a +/// tenant's stored console override and credential already live at +/// [`RUNTIME_CONFIG_KEY`] / [`KEY_KEY`], and the store has no rename — so +/// namespacing every harness would silently orphan the config of every company +/// already running, which is the one migration this design cannot afford. +/// +/// Non-default harnesses namespace under `harness//`, so two `built_in` +/// harnesses can hold two different OpenRouter accounts. +pub fn runtime_config_key(harness_id: &str, is_default: bool) -> String { + if is_default { + return RUNTIME_CONFIG_KEY.to_string(); + } + format!("harness/{harness_id}/{RUNTIME_CONFIG_KEY}") +} + +/// The credential key for `harness_id`. Same default-harness rule as +/// [`runtime_config_key`]. +pub fn harness_key_key(harness_id: &str, is_default: bool) -> String { + if is_default { + return KEY_KEY.to_string(); + } + format!("harness/{harness_id}/{KEY_KEY}") +} + +/// Which harness's secrets a resolution reads, and whether that harness is the +/// company default (which keeps the flat legacy keys). +/// +/// Passed as one value rather than two loose arguments because the pair is only +/// ever meaningful together — an id without the default flag cannot name a key. +#[derive(Clone, Debug)] +pub struct HarnessScope { + /// The harness id. + pub id: String, + /// Whether it is the company's default harness. + pub is_default: bool, +} + +impl HarnessScope { + /// The scope for a company's default harness — what every pre-existing + /// caller means, and what keeps them reading the flat keys. + pub fn default_harness(id: impl Into) -> Self { + Self { + id: id.into(), + is_default: true, + } + } + + /// A named, non-default harness. + pub fn named(id: impl Into) -> Self { + Self { + id: id.into(), + is_default: false, + } + } + + /// This scope's runtime-config secret key. + pub fn config_key(&self) -> String { + runtime_config_key(&self.id, self.is_default) + } + + /// This scope's credential secret key. + pub fn key_key(&self) -> String { + harness_key_key(&self.id, self.is_default) + } +} + +impl Default for HarnessScope { + fn default() -> Self { + Self::default_harness(crate::company::types::IMPLICIT_HARNESS_ID) + } +} + +/// The platform's OpenAI-compatible endpoint — the subscription proxy an +/// `openrouter` company with **no** key of its own resolves against. +/// +/// The proxy fronts OpenRouter upstream and meters the spend against the +/// tenant's subscription, so from the workload's point of view this and +/// [`OPENROUTER_BASE_URL`] serve the same catalogue; only who pays differs. +pub const PLATFORM_BASE_URL: &str = "https://api.tinyhumans.ai/openai/v1"; + +/// The provider kind removed when OpenCompany stopped exposing its own model +/// SKUs. A manifest or stored runtime blob still naming it aliases to +/// [`DEFAULT_PROVIDER`] rather than failing: a runtime blob is data an operator +/// cannot hand-edit, so hard-failing on it would strand a tenant whose console +/// wrote a value that used to be valid. +pub const LEGACY_MANAGED: &str = "managed"; + +/// The provider a company gets when nothing names one. +pub const DEFAULT_PROVIDER: &str = "openrouter"; + +/// Default concrete OpenRouter model id per abstract tier. +/// +/// Used on the **direct** path only. A tier names a workload, and something has +/// to turn it into a model id before the request leaves this process — but only +/// when the endpoint would not do it. See [`model_for_tier`]. +/// +/// The slugs mirror the platform's own OpenRouter bindings, so proxied and +/// direct resolve to the same models by default. +pub const DEFAULT_TIER_MODELS: &[(&str, &str)] = &[ + ("chat-v1", "deepseek/deepseek-v4-flash"), + ("reasoning-v1", "deepseek/deepseek-v4-pro"), + ("agentic-v1", "deepseek/deepseek-v4-pro"), + ("vision-v1", "qwen/qwen3.7-plus"), +]; + +/// The concrete model id to put on the wire for `tier`. +/// +/// **The two paths need different answers, and sending the wrong one fails.** +/// +/// * **Proxied** — the platform endpoint resolves tier names itself, against a +/// curated registry that pins each tier to a sub-provider so its rate card +/// stays exact. A bare tier is exactly what it wants. It also accepts a +/// concrete model, but only under its own `openrouter//` +/// namespace, and only when passthrough is switched on there — which it is +/// not by default. So a tier is the only thing that always works. +/// * **Direct** — OpenRouter has never heard of `chat-v1`, so a tier must be +/// resolved here or the request 400s. +/// +/// An operator's own `models` entry is honoured verbatim on both paths: they +/// named a specific model and it is not this function's place to rewrite it +/// (on the proxied path they can write the `openrouter/…` form themselves). +pub fn model_for_tier(tier: &str, overrides: &BTreeMap, proxied: bool) -> String { + if let Some(mapped) = overrides.get(tier) { + return mapped.clone(); + } + if proxied { + // Let the platform resolve it. Substituting a concrete slug here would + // bypass its per-tier provider pinning and, with passthrough off, be + // rejected outright. + return tier.to_string(); + } + DEFAULT_TIER_MODELS + .iter() + .find(|(name, _)| *name == tier) + .map(|(_, model)| (*model).to_string()) + .unwrap_or_else(|| tier.to_string()) +} + +/// Normalizes a provider kind: blank and the legacy `managed` both become +/// [`DEFAULT_PROVIDER`]; anything else passes through for validation to judge. +pub fn normalize_provider(provider: &str) -> &str { + match provider.trim() { + "" | LEGACY_MANAGED => DEFAULT_PROVIDER, + other => other, + } +} /// OpenRouter's OpenAI-compatible base URL — used when the `openrouter` /// provider names no explicit `base_url`. @@ -120,6 +266,9 @@ pub struct InferenceDecl { /// The outbound credential. Private — read only through /// [`bearer`](Self::bearer); never serialized. credential: Credential, + /// Whether this rides the platform's subscription proxy. Read through + /// [`is_proxied`](Self::is_proxied). + proxied: bool, } impl InferenceDecl { @@ -145,57 +294,103 @@ impl InferenceDecl { self.credential.configured() } - /// The stable telemetry slug for this provider (`managed` / `openrouter` / - /// `byok` / `ollama`). + /// Whether this config rides the platform's subscription proxy rather than + /// a credential the tenant supplied. + /// + /// True only for `openrouter` with no tenant key — the default a company + /// starts on. It is what separates "the subscription pays" from "the tenant + /// pays", which is why it is recorded here rather than re-derived from the + /// base URL by every caller that cares. + pub fn is_proxied(&self) -> bool { + self.proxied + } + + /// The stable telemetry slug for this config + /// (`subscription` / `openrouter` / `byok` / `ollama`). + /// + /// Distinguishes proxied from direct OpenRouter, because those are two + /// different payers and a Usage view that merged them would be telling the + /// operator nothing. pub fn telemetry_slug(&self) -> &'static str { + if self.proxied { + return "subscription"; + } provider_slug(&self.provider) } } -/// The stable telemetry slug for a provider kind. Unknown kinds fall back to -/// `managed` (the safe default attribution). +/// The stable telemetry slug for a provider kind. +/// +/// An unknown kind reports `unknown` rather than being folded into a real +/// provider's attribution: [`resolve_effective`] rejects one outright, so +/// reaching here with one means something upstream is wrong, and quietly +/// billing it to a provider that was never called would hide that. +/// +/// This answers on the *kind* alone. Proxied OpenRouter slugs as +/// `subscription`, which needs the credential too — see +/// [`InferenceDecl::telemetry_slug`]. pub fn provider_slug(provider: &str) -> &'static str { - match provider.trim() { + match normalize_provider(provider) { "openrouter" => "openrouter", "ollama" => "ollama", "openai_compatible" => "byok", - _ => "managed", + _ => "unknown", } } -/// Resolves the effective `(base_url, credential)` for a provider. +/// Resolves the effective `(base_url, credential, proxied)` for a provider. +/// +/// **`openrouter` is dual-mode**, and that is the whole shape of the product's +/// first-run story: /// -/// The `managed` kind is special: it is the *platform* brain, so it inherits the -/// env default's base URL and credential when a source (manifest/runtime) names -/// `managed` without supplying its own — otherwise a hand-written -/// `provider = "managed"` would drop the platform key and 401. Every other kind -/// uses its own configured base URL + key verbatim. +/// * **No tenant key** — the config inherits the platform endpoint and the +/// platform credential, and the subscription pays. This is where a company +/// starts, with nothing configured and nobody asked for a card. +/// * **A tenant `sk-or-…`** — the config goes direct to OpenRouter on the +/// tenant's own account. +/// +/// The inheritance branch is the one `managed` used to own, and it moved here +/// rather than being deleted for the same reason it existed: a config that named +/// a provider but dropped the platform key would 401 rather than fall back. +/// +/// The one exception is a keyless `openrouter` that also sets its own +/// `base_url`: that endpoint is not the platform's, so the platform credential +/// is withheld and the config goes direct (keyless) instead — sending the +/// platform token to an arbitrary override would leak it. +/// +/// Every other kind uses its own configured base URL and key verbatim — those +/// are third-party endpoints we hold no credential for. fn resolve_endpoint( provider: &str, base_url_override: Option<&str>, key: String, env_default: Option<&EnvDefault>, -) -> (String, Credential) { +) -> (String, Credential, bool) { let base_url_override = base_url_override.map(str::trim).filter(|s| !s.is_empty()); - if provider.trim() == "managed" { - let base_url = base_url_override - .map(str::to_string) - .or_else(|| env_default.map(|e| e.base_url.clone())) - .unwrap_or_else(|| MANAGED_BASE_URL.to_string()); - let credential = if !key.trim().is_empty() { - Credential::from_value(key) - } else { - env_default - .map(|e| e.credential.clone()) - .unwrap_or(Credential::None) - }; - (base_url, credential) - } else { - ( - effective_base_url(provider, base_url_override), - Credential::from_value(key), - ) + let has_key = !key.trim().is_empty(); + + if normalize_provider(provider) == "openrouter" && !has_key { + // The platform credential rides only the platform's own endpoint. A + // tenant-supplied base URL override with no key is a direct (keyless) + // config, not the inheritance branch: pairing the platform token with + // an arbitrary endpoint would leak it to wherever the override points. + if let Some(base_url) = base_url_override { + return (base_url.to_string(), Credential::None, false); + } + let base_url = env_default + .map(|e| e.base_url.clone()) + .unwrap_or_else(|| PLATFORM_BASE_URL.to_string()); + let credential = env_default + .map(|e| e.credential.clone()) + .unwrap_or(Credential::None); + return (base_url, credential, true); } + + ( + effective_base_url(provider, base_url_override), + Credential::from_value(key), + false, + ) } /// A declaration for the **first-run** connection test, before any company @@ -225,7 +420,7 @@ pub fn decl_for_probe( env_default: Option<&EnvDefault>, ) -> InferenceDecl { let provider = provider.trim().to_string(); - let (base_url, credential) = resolve_endpoint( + let (base_url, credential, proxied) = resolve_endpoint( &provider, base_url, key.unwrap_or_default().trim().to_string(), @@ -237,22 +432,25 @@ pub fn decl_for_probe( models: BTreeMap::new(), source: InferenceSource::Runtime, credential, + proxied, } } /// The effective base URL for a provider kind, given an optional override. /// -/// `managed`/`openrouter` default to their well-known endpoints; `ollama` -/// backstops to a local default; `openai_compatible` has no default (validation -/// requires an explicit URL). +/// This is the **direct** endpoint for each kind. Proxied `openrouter` does not +/// come through here — it inherits the platform endpoint in +/// [`resolve_endpoint`], which is the only place that distinction is made. +/// +/// `ollama` backstops to a local default; `openai_compatible` has no default +/// (validation requires an explicit URL). pub fn effective_base_url(provider: &str, override_url: Option<&str>) -> String { let override_url = override_url.map(str::trim).filter(|s| !s.is_empty()); - match provider.trim() { - "openrouter" => override_url.unwrap_or(OPENROUTER_BASE_URL).to_string(), + match normalize_provider(provider) { "ollama" => override_url.unwrap_or(OLLAMA_DEFAULT_BASE_URL).to_string(), "openai_compatible" => override_url.unwrap_or_default().to_string(), - // managed (and any unknown kind, which validation rejects separately). - _ => override_url.unwrap_or(MANAGED_BASE_URL).to_string(), + // openrouter, and any unknown kind (which `resolve_effective` rejects). + _ => override_url.unwrap_or(OPENROUTER_BASE_URL).to_string(), } } @@ -262,7 +460,16 @@ pub async fn load_runtime_config( company: &CompanyId, secrets: &dyn SecretStore, ) -> Result> { - let Some(SecretValue(raw)) = secrets.get(company, RUNTIME_CONFIG_KEY).await? else { + load_runtime_config_scoped(company, secrets, &HarnessScope::default()).await +} + +/// [`load_runtime_config`] for one harness's own slot. +pub async fn load_runtime_config_scoped( + company: &CompanyId, + secrets: &dyn SecretStore, + scope: &HarnessScope, +) -> Result> { + let Some(SecretValue(raw)) = secrets.get(company, &scope.config_key()).await? else { return Ok(None); }; if raw.trim().is_empty() { @@ -282,11 +489,21 @@ pub async fn save_runtime_config( company: &CompanyId, secrets: &dyn SecretStore, config: &RuntimeInference, +) -> Result<()> { + save_runtime_config_scoped(company, secrets, config, &HarnessScope::default()).await +} + +/// [`save_runtime_config`] for one harness's own slot. +pub async fn save_runtime_config_scoped( + company: &CompanyId, + secrets: &dyn SecretStore, + config: &RuntimeInference, + scope: &HarnessScope, ) -> Result<()> { let raw = serde_json::to_string(config) .map_err(|e| OpenCompanyError::Store(format!("serializing inference config: {e}")))?; secrets - .set(company, RUNTIME_CONFIG_KEY, SecretValue(raw)) + .set(company, &scope.config_key(), SecretValue(raw)) .await } @@ -294,8 +511,17 @@ pub async fn save_runtime_config( /// manifest/managed). Best-effort — the store has no delete, so an empty value /// reads back as unset. pub async fn clear_runtime_config(company: &CompanyId, secrets: &dyn SecretStore) -> Result<()> { + clear_runtime_config_scoped(company, secrets, &HarnessScope::default()).await +} + +/// [`clear_runtime_config`] for one harness's own slot. +pub async fn clear_runtime_config_scoped( + company: &CompanyId, + secrets: &dyn SecretStore, + scope: &HarnessScope, +) -> Result<()> { secrets - .set(company, RUNTIME_CONFIG_KEY, SecretValue(String::new())) + .set(company, &scope.config_key(), SecretValue(String::new())) .await } @@ -310,7 +536,17 @@ pub async fn load_key( secrets: &dyn SecretStore, override_key: Option<&str>, ) -> Result { - if let Some(SecretValue(raw)) = secrets.get(company, KEY_KEY).await? + load_key_scoped(company, secrets, override_key, &HarnessScope::default()).await +} + +/// [`load_key`] for one harness's own credential slot. +pub async fn load_key_scoped( + company: &CompanyId, + secrets: &dyn SecretStore, + override_key: Option<&str>, + scope: &HarnessScope, +) -> Result { + if let Some(SecretValue(raw)) = secrets.get(company, &scope.key_key()).await? && !raw.trim().is_empty() { return Ok(raw); @@ -326,8 +562,18 @@ pub async fn load_key( /// Writes the company's outbound inference credential (write-only intake). pub async fn store_key(company: &CompanyId, secrets: &dyn SecretStore, key: &str) -> Result<()> { + store_key_scoped(company, secrets, key, &HarnessScope::default()).await +} + +/// [`store_key`] for one harness's own credential slot. +pub async fn store_key_scoped( + company: &CompanyId, + secrets: &dyn SecretStore, + key: &str, + scope: &HarnessScope, +) -> Result<()> { secrets - .set(company, KEY_KEY, SecretValue(key.to_string())) + .set(company, &scope.key_key(), SecretValue(key.to_string())) .await } @@ -366,12 +612,40 @@ pub async fn resolve_effective( manifest: &Inference, env_default: Option<&EnvDefault>, secrets: &dyn SecretStore, +) -> Result> { + resolve_effective_scoped( + company, + manifest, + env_default, + secrets, + &HarnessScope::default(), + ) + .await +} + +/// [`resolve_effective`] against one harness's own config and credential slots. +/// +/// `manifest` is that harness's `[harness.inference]`, or the company-level +/// `[inference]` when it declares none — the caller picks, because only it knows +/// which fallback applies. +/// +/// The precedence within a harness is unchanged (runtime > manifest > env +/// default); what differs is only *which* secret keys the runtime and credential +/// tiers read. Two `built_in` harnesses therefore resolve independently, which +/// is what lets one run on the subscription while the other runs on a key. +pub async fn resolve_effective_scoped( + company: &CompanyId, + manifest: &Inference, + env_default: Option<&EnvDefault>, + secrets: &dyn SecretStore, + scope: &HarnessScope, ) -> Result> { // 1. Runtime override (console) wins. - if let Some(runtime) = load_runtime_config(company, secrets).await? { - let provider = runtime.provider.trim().to_string(); - let key = load_key(company, secrets, None).await?; - let (base_url, credential) = + if let Some(runtime) = load_runtime_config_scoped(company, secrets, scope).await? { + let provider = normalize_provider(&runtime.provider).to_string(); + reject_unknown_provider(&provider, "the stored runtime inference config")?; + let key = load_key_scoped(company, secrets, None, scope).await?; + let (base_url, credential, proxied) = resolve_endpoint(&provider, runtime.base_url.as_deref(), key, env_default); return Ok(Some(InferenceDecl { provider, @@ -379,19 +653,18 @@ pub async fn resolve_effective( models: runtime.models, source: InferenceSource::Runtime, credential, + proxied, })); } // 2. Manifest `[inference]`. if manifest.is_set() { - let provider = manifest - .provider - .as_deref() - .unwrap_or_default() - .trim() - .to_string(); - let key = load_key(company, secrets, manifest.api_key_secret.as_deref()).await?; - let (base_url, credential) = + let provider = + normalize_provider(manifest.provider.as_deref().unwrap_or_default()).to_string(); + reject_unknown_provider(&provider, "`[inference].provider`")?; + let key = + load_key_scoped(company, secrets, manifest.api_key_secret.as_deref(), scope).await?; + let (base_url, credential, proxied) = resolve_endpoint(&provider, manifest.base_url.as_deref(), key, env_default); return Ok(Some(InferenceDecl { provider, @@ -399,23 +672,54 @@ pub async fn resolve_effective( models: manifest.models.clone(), source: InferenceSource::Manifest, credential, + proxied, })); } - // 3. Platform-injected managed default. + // 3. The platform-injected default: OpenRouter, proxied on the subscription. + // A company that has configured nothing lands here, which is why it must + // be a working config and not a prompt for a credential. + // + // It still runs through `resolve_endpoint` rather than assuming proxied, + // because a console-set key is a configuration act even when the operator + // never named a provider: the console's key field on a fresh company + // writes `inference/key` and nothing else. Assuming proxied here would + // take that key, store it, report it as configured — and then never send + // it anywhere. if let Some(env) = env_default { + let key = load_key_scoped(company, secrets, None, scope).await?; + let (base_url, credential, proxied) = + resolve_endpoint(DEFAULT_PROVIDER, None, key, Some(env)); return Ok(Some(InferenceDecl { - provider: "managed".to_string(), - base_url: env.base_url.clone(), + provider: DEFAULT_PROVIDER.to_string(), + base_url, models: BTreeMap::new(), source: InferenceSource::Default, - credential: env.credential.clone(), + credential, + proxied, })); } Ok(None) } +/// Fails a provider kind that is not in [`INFERENCE_PROVIDERS`]. +/// +/// The manifest validator already rejects one, but a **stored runtime blob** +/// never passes through it: the console wrote it, possibly under an older build +/// whose vocabulary differed. Resolving one silently would attribute its spend to +/// whatever the fallback happened to be, so it fails loudly here instead — the +/// one place both sources converge. +fn reject_unknown_provider(provider: &str, whence: &str) -> Result<()> { + if crate::company::types::INFERENCE_PROVIDERS.contains(&provider) { + return Ok(()); + } + Err(OpenCompanyError::Config(format!( + "{whence} names an unknown inference provider `{provider}` — expected one of {}.", + crate::company::types::INFERENCE_PROVIDERS.join(", ") + ))) +} + /// Validates the manifest `[inference]` section, returning every problem in /// prosumer language. An absent section (`provider = None`) is inert. Shared by /// manifest validation and the ops `PUT` route (via [`validate_runtime`]). @@ -449,6 +753,13 @@ fn validate_parts( ) -> Vec { let mut problems = Vec::new(); + // `managed` aliases rather than failing. It named a real thing until + // OpenCompany stopped exposing its own SKUs, and a committed manifest that + // still says it means "the platform's brain" — which is now proxied + // OpenRouter. Rejecting it would break bundles that were valid when written, + // to no purpose: the intent still resolves. + let provider = normalize_provider(provider); + if !INFERENCE_PROVIDERS.contains(&provider) { problems.push(format!( "`[inference].provider` must be one of {} — you wrote `{provider}`.", @@ -575,7 +886,9 @@ mod tests { .unwrap() .expect("env default resolves"); assert_eq!(decl.source, InferenceSource::Default); - assert_eq!(decl.provider, "managed"); + assert_eq!(decl.provider, DEFAULT_PROVIDER); + assert!(decl.is_proxied(), "the default rides the subscription"); + assert_eq!(decl.telemetry_slug(), "subscription"); assert_eq!(bearer(&decl).await.as_deref(), Some("env-key")); // Manifest beats env. @@ -607,47 +920,165 @@ mod tests { assert_eq!(decl.source, InferenceSource::Runtime); assert_eq!(decl.provider, "openrouter"); assert_eq!(decl.base_url, OPENROUTER_BASE_URL); + assert!(!decl.is_proxied(), "a tenant key goes direct"); + assert_eq!(decl.telemetry_slug(), "openrouter"); assert_eq!(bearer(&decl).await.as_deref(), Some("or-secret")); assert!(decl.key_configured()); } + /// A keyless `openrouter` inherits the platform endpoint and credential + /// rather than dropping them — the branch `managed` used to own. Without it a + /// company that names its provider but holds no key of its own would 401 + /// instead of riding the subscription. #[tokio::test] - async fn manifest_managed_inherits_env_credential() { - // A hand-written `provider = "managed"` must still use the platform - // env key + base URL rather than dropping the credential. + async fn keyless_openrouter_inherits_the_platform_endpoint_and_credential() { let company = CompanyId::new("acme"); let secrets = MemSecrets::default(); let env = EnvDefault { base_url: "https://env.example/openai/v1".into(), credential: Credential::from_value("platform-key"), }; - let decl = resolve_effective(&company, &inference("managed"), Some(&env), &secrets) + let decl = resolve_effective(&company, &inference("openrouter"), Some(&env), &secrets) .await .unwrap() .unwrap(); assert_eq!(decl.source, InferenceSource::Manifest); - assert_eq!(decl.provider, "managed"); + assert_eq!(decl.provider, "openrouter"); assert_eq!(decl.base_url, "https://env.example/openai/v1"); + assert!(decl.is_proxied()); assert_eq!(bearer(&decl).await.as_deref(), Some("platform-key")); } + /// A keyless `openrouter` with a tenant-supplied `base_url` override goes + /// direct with **no** credential — the platform token must not ride an + /// arbitrary endpoint the operator pointed it at. #[tokio::test] - async fn console_key_beats_env_for_the_managed_provider() { - // Issue #585: the company's own key is the admin's to set. A key stored - // through the console must win over the deploy-time env credential even - // on the `managed` provider — otherwise the only way to pay for a tenant - // is an environment variable the admin cannot reach. + async fn keyless_openrouter_never_sends_the_platform_credential_to_an_override() { let company = CompanyId::new("acme"); let secrets = MemSecrets::default(); let env = EnvDefault { base_url: "https://env.example/openai/v1".into(), credential: Credential::from_value("platform-key"), }; + let mut manifest = inference("openrouter"); + manifest.base_url = Some("https://attacker.example/v1".into()); + let decl = resolve_effective(&company, &manifest, Some(&env), &secrets) + .await + .unwrap() + .unwrap(); + assert_eq!(decl.base_url, "https://attacker.example/v1"); + assert!( + !decl.is_proxied(), + "an arbitrary endpoint is not the subscription" + ); + assert!( + !decl.key_configured(), + "a keyless config holds no credential to send" + ); + assert_eq!(decl.telemetry_slug(), "openrouter"); + assert_eq!( + bearer(&decl).await, + None, + "the platform credential stays home" + ); + } + + /// A committed manifest still saying `provider = "managed"` resolves as + /// proxied OpenRouter rather than failing. It was valid when written, and the + /// intent — "the platform's brain" — is exactly what proxied OpenRouter is. + #[tokio::test] + async fn a_legacy_managed_manifest_aliases_to_proxied_openrouter() { + let company = CompanyId::new("acme"); + let secrets = MemSecrets::default(); + let env = EnvDefault { + base_url: "https://env.example/openai/v1".into(), + credential: Credential::from_value("platform-key"), + }; + let decl = resolve_effective(&company, &inference(LEGACY_MANAGED), Some(&env), &secrets) + .await + .unwrap() + .unwrap(); + assert_eq!(decl.provider, DEFAULT_PROVIDER); + assert!(decl.is_proxied()); + assert_eq!(decl.base_url, "https://env.example/openai/v1"); + assert_eq!(bearer(&decl).await.as_deref(), Some("platform-key")); + assert!( + validate_inference(&inference(LEGACY_MANAGED)).is_empty(), + "and it still validates" + ); + + // The same alias applies to a stored runtime blob, which an operator + // cannot hand-edit — the case that would otherwise strand a tenant. save_runtime_config( &company, &secrets, &RuntimeInference { - provider: "managed".into(), + provider: LEGACY_MANAGED.into(), + base_url: None, + models: BTreeMap::new(), + }, + ) + .await + .unwrap(); + let decl = resolve_effective(&company, &Inference::default(), Some(&env), &secrets) + .await + .unwrap() + .unwrap(); + assert_eq!(decl.source, InferenceSource::Runtime); + assert_eq!(decl.provider, DEFAULT_PROVIDER); + assert!(decl.is_proxied()); + } + + /// A stored runtime blob naming a provider this build does not know fails + /// loudly rather than resolving to whatever the fallback happened to be. + #[tokio::test] + async fn an_unknown_stored_provider_is_an_error_not_a_silent_fallback() { + let company = CompanyId::new("acme"); + let secrets = MemSecrets::default(); + save_runtime_config( + &company, + &secrets, + &RuntimeInference { + provider: "telepathy".into(), + base_url: None, + models: BTreeMap::new(), + }, + ) + .await + .unwrap(); + let err = resolve_effective(&company, &Inference::default(), None, &secrets) + .await + .expect_err("unknown provider must fail"); + let msg = err.to_string(); + assert!(msg.contains("telepathy"), "{msg}"); + assert!(msg.contains("openrouter"), "names what is valid: {msg}"); + } + + /// Issue #585: the company's own key is the admin's to set, and a key stored + /// through the console wins over the deploy-time env credential — otherwise + /// the only way to pay for a tenant is an environment variable the admin + /// cannot reach. + /// + /// **What changed with `managed`'s removal.** Under `managed`, a console key + /// kept the *platform* endpoint, so an admin could bill their own account + /// through the proxy. `openrouter` is dual-mode instead: a key means an + /// OpenRouter key, so it goes direct to OpenRouter — sending an `sk-or-…` to + /// the platform proxy would simply be rejected. An admin who wants the + /// platform endpoint with a credential of their own now names + /// `openai_compatible` with that `base_url`. + #[tokio::test] + async fn a_console_key_wins_over_the_env_credential_and_goes_direct() { + let company = CompanyId::new("acme"); + let secrets = MemSecrets::default(); + let env = EnvDefault { + base_url: "https://env.example/openai/v1".into(), + credential: Credential::from_value("platform-key"), + }; + save_runtime_config( + &company, + &secrets, + &RuntimeInference { + provider: "openrouter".into(), base_url: None, models: BTreeMap::new(), }, @@ -659,21 +1090,23 @@ mod tests { let decl = resolve_effective(&company, &Inference::default(), Some(&env), &secrets) .await .unwrap() - .expect("runtime managed resolves"); + .expect("runtime openrouter resolves"); assert_eq!(decl.source, InferenceSource::Runtime); - assert_eq!(decl.provider, "managed"); + assert_eq!(decl.provider, "openrouter"); assert_eq!(bearer(&decl).await.as_deref(), Some("company-key")); assert!(decl.key_configured()); - // The endpoint still inherits the platform's, so setting only a key does - // not silently move the tenant off the managed brain. - assert_eq!(decl.base_url, "https://env.example/openai/v1"); + assert!(!decl.is_proxied(), "the tenant's own account pays"); + assert_eq!(decl.base_url, OPENROUTER_BASE_URL); - // Clearing it falls back to the env credential rather than 401ing. + // Clearing it falls back to the subscription rather than 401ing — the + // property that makes a key genuinely optional in both directions. clear_key(&company, &secrets).await.unwrap(); let decl = resolve_effective(&company, &Inference::default(), Some(&env), &secrets) .await .unwrap() - .expect("runtime managed still resolves"); + .expect("still resolves with no key"); + assert!(decl.is_proxied()); + assert_eq!(decl.base_url, "https://env.example/openai/v1"); assert_eq!(bearer(&decl).await.as_deref(), Some("platform-key")); } @@ -924,18 +1357,176 @@ mod tests { ); } + /// The isolation property named harnesses exist for: two `built_in` + /// harnesses on one company resolve independently, so one can ride the + /// subscription while the other runs on a key of its own. + #[tokio::test] + async fn two_harnesses_on_one_company_resolve_independently() { + let company = CompanyId::new("acme"); + let secrets = MemSecrets::default(); + let env = EnvDefault { + base_url: "https://env.example/v1".into(), + credential: Credential::from_value("platform-key"), + }; + let embedded = HarnessScope::default_harness("embedded"); + let deep = HarnessScope::named("deep"); + + // Only `deep` gets a key. + store_key_scoped(&company, &secrets, "sk-or-deep", &deep) + .await + .unwrap(); + + let d = resolve_effective_scoped( + &company, + &inference("openrouter"), + Some(&env), + &secrets, + &deep, + ) + .await + .unwrap() + .unwrap(); + assert!(!d.is_proxied(), "deep pays its own way"); + assert_eq!(d.base_url, OPENROUTER_BASE_URL); + assert_eq!(bearer(&d).await.as_deref(), Some("sk-or-deep")); + + let e = resolve_effective_scoped( + &company, + &inference("openrouter"), + Some(&env), + &secrets, + &embedded, + ) + .await + .unwrap() + .unwrap(); + assert!(e.is_proxied(), "embedded is untouched by deep's key"); + assert_eq!(e.base_url, "https://env.example/v1"); + assert_eq!(bearer(&e).await.as_deref(), Some("platform-key")); + } + + /// The default harness keeps the flat legacy keys, so a tenant whose console + /// already wrote `inference/key` keeps working with no migration — the store + /// has no rename, so getting this wrong would orphan every running company. + #[tokio::test] + async fn the_default_harness_reads_the_legacy_flat_keys() { + let company = CompanyId::new("acme"); + let secrets = MemSecrets::default(); + + // Written the pre-harness way. + store_key(&company, &secrets, "legacy-key").await.unwrap(); + save_runtime_config( + &company, + &secrets, + &RuntimeInference { + provider: "openrouter".into(), + base_url: None, + models: BTreeMap::new(), + }, + ) + .await + .unwrap(); + + // Read back through the scoped path, as the default harness. + let scope = HarnessScope::default_harness("embedded"); + assert_eq!(scope.key_key(), KEY_KEY); + assert_eq!(scope.config_key(), RUNTIME_CONFIG_KEY); + + let decl = + resolve_effective_scoped(&company, &Inference::default(), None, &secrets, &scope) + .await + .unwrap() + .expect("the legacy config resolves"); + assert_eq!(decl.source, InferenceSource::Runtime); + assert_eq!(bearer(&decl).await.as_deref(), Some("legacy-key")); + + // A named harness namespaces instead, and sees none of it. + let named = HarnessScope::named("deep"); + assert_eq!(named.key_key(), "harness/deep/inference/key"); + assert_eq!(named.config_key(), "harness/deep/inference/config"); + assert!( + load_runtime_config_scoped(&company, &secrets, &named) + .await + .unwrap() + .is_none() + ); + } + + /// Every declared tier resolves to a concrete OpenRouter slug, and a tier + /// left unmapped by the harness still does. + /// + /// This is what makes the DIRECT path work at all: OpenRouter has never + /// heard of `chat-v1`, so a bare tier on a tenant's own key would 400. + #[test] + fn every_tier_resolves_to_a_concrete_model_id_on_the_direct_path() { + let none = BTreeMap::new(); + for tier in crate::company::types::INFERENCE_TIERS { + let resolved = model_for_tier(tier, &none, false); + assert_ne!( + &resolved, tier, + "`{tier}` must map to a concrete slug, not pass through" + ); + assert!( + resolved.contains('/'), + "`{tier}` resolved to `{resolved}`, which is not an OpenRouter slug" + ); + } + } + + #[test] + fn a_harness_override_beats_the_default_and_a_concrete_slug_passes_through() { + let overrides = + BTreeMap::from([("chat-v1".to_string(), "anthropic/claude-haiku".to_string())]); + // An operator's own entry is honoured verbatim on BOTH paths — they + // named a specific model, and rewriting it is not this function's call. + for proxied in [false, true] { + assert_eq!( + model_for_tier("chat-v1", &overrides, proxied), + "anthropic/claude-haiku" + ); + } + // An unmapped tier still takes the shipped default on the direct path. + assert_eq!( + model_for_tier("reasoning-v1", &overrides, false), + "deepseek/deepseek-v4-pro" + ); + // A caller naming a concrete slug is not treated as an unknown tier. + assert_eq!( + model_for_tier("anthropic/claude-sonnet-4.5", &BTreeMap::new(), false), + "anthropic/claude-sonnet-4.5" + ); + } + + /// The proxied path keeps the tier name. The platform's registry routes on + /// it and pins each tier to a sub-provider; substituting a concrete slug + /// would bypass that pinning, and its passthrough namespace is opt-in and + /// off by default, so the slug would simply be rejected. + #[test] + fn the_proxied_path_keeps_the_tier_name() { + let none = BTreeMap::new(); + for tier in crate::company::types::INFERENCE_TIERS { + assert_eq!(&model_for_tier(tier, &none, true), tier); + } + } + #[test] fn provider_slugs_map_as_documented() { - assert_eq!(provider_slug("managed"), "managed"); assert_eq!(provider_slug("openrouter"), "openrouter"); assert_eq!(provider_slug("openai_compatible"), "byok"); assert_eq!(provider_slug("ollama"), "ollama"); - assert_eq!(provider_slug("mystery"), "managed"); + // The legacy kind slugs as what it now is, so historical usage rows and + // new ones aggregate together. + assert_eq!(provider_slug(LEGACY_MANAGED), "openrouter"); + // An unknown kind is never folded into a real provider's attribution. + assert_eq!(provider_slug("mystery"), "unknown"); } #[test] fn effective_base_url_defaults_per_provider() { - assert_eq!(effective_base_url("managed", None), MANAGED_BASE_URL); + assert_eq!( + effective_base_url(LEGACY_MANAGED, None), + OPENROUTER_BASE_URL + ); assert_eq!(effective_base_url("openrouter", None), OPENROUTER_BASE_URL); assert_eq!(effective_base_url("ollama", None), OLLAMA_DEFAULT_BASE_URL); assert_eq!( diff --git a/src/company/manifest.rs b/src/company/manifest.rs index 4c066a048..b7497b5d2 100644 --- a/src/company/manifest.rs +++ b/src/company/manifest.rs @@ -5,6 +5,7 @@ //! the manifest inside a company directory, preferring `company.toml` over the //! legacy `agents.toml`. +use std::collections::BTreeSet; use std::fmt::Write as _; use std::path::{Path, PathBuf}; @@ -12,9 +13,10 @@ use crate::error::{OpenCompanyError, Result}; use crate::ports::decode_wallet_address; use super::types::{ - AUTH_MODES, BRAIN_MODES, CONNECTION_PRIORITIES, CompanyManifest, GATEABLE_NAMESPACES, - KNOWN_CHANNELS, MAX_DELEGATION_DEPTH_BOUNDS, PLAN_NAMES, PLAN_PERIODS, POLICY_MODES, - PROMPT_CLASSES, TIERS, TOOL_PROVIDERS, + ACP_AGENTS, ACP_TRANSPORTS, AUTH_MODES, BRAIN_MODES, CONNECTION_PRIORITIES, CompanyManifest, + GATEABLE_NAMESPACES, HARNESS_KINDS, Harness, IMPLICIT_HARNESS_ID, Inference, KNOWN_CHANNELS, + MAX_DELEGATION_DEPTH_BOUNDS, PLAN_NAMES, PLAN_PERIODS, POLICY_MODES, PROMPT_CLASSES, TIERS, + TOOL_PROVIDERS, }; /// The `delegates_to` entry that means "every desk this company has". @@ -70,6 +72,68 @@ pub fn discover(input: &Path) -> Result { } impl CompanyManifest { + /// The company's harnesses, with the implicit one synthesized when the + /// manifest declares none. + /// + /// **Read harnesses through this, never through the `harnesses` field.** A + /// company with no `[[harness]]` block still runs on a harness — the + /// `built_in` one on the company-level `[inference]` — and a caller that + /// looked at the bare field would see an empty list and conclude the company + /// has no engine, which is never true. + pub fn effective_harnesses(&self) -> Vec { + if self.harnesses.is_empty() { + return vec![Harness::implicit()]; + } + self.harnesses.clone() + } + + /// The id of the harness agents naming none run on. + /// + /// The entry marked `default = true`; the first declared if validation was + /// skipped and none is marked, so this is total rather than panicking on a + /// manifest that reached here unvalidated. + pub fn default_harness_id(&self) -> String { + let harnesses = self.effective_harnesses(); + harnesses + .iter() + .find(|h| h.default) + .or_else(|| harnesses.first()) + .map(|h| h.id.clone()) + .unwrap_or_else(|| IMPLICIT_HARNESS_ID.to_string()) + } + + /// The default harness's `[harness.inference]`, when that harness declares + /// one; `None` when it runs on the company-level `[inference]`. + /// + /// The default harness is the one the base provider resolves for, and its + /// own inference section must beat the company-level one — the same + /// precedence a named harness gets in [`lanes::build`](crate::harness::lanes::build). + /// `None` (not "empty") because an absent declaration means "fall back to + /// `[inference]`", which the caller already holds. + pub fn default_harness_inference(&self) -> Option { + let default_id = self.default_harness_id(); + self.effective_harnesses() + .into_iter() + .find(|h| h.id == default_id) + .and_then(|h| h.inference) + } + + /// The harness `agent_id` runs on, resolving an unset binding to the + /// default. `None` only when the named harness does not exist — which + /// [`validate`](Self::validate) rejects, so a validated manifest always + /// answers. + pub fn harness_for(&self, agent_id: &str) -> Option { + let named = self + .agents + .iter() + .find(|a| a.id == agent_id) + .and_then(|a| a.harness.clone()); + let want = named.unwrap_or_else(|| self.default_harness_id()); + self.effective_harnesses() + .into_iter() + .find(|h| h.id == want) + } + /// Reads, parses, and validates a manifest from `path`. /// /// `path` may be a manifest file or a directory containing one. Validation @@ -419,6 +483,8 @@ impl CompanyManifest { problems.push(one_of("`[brain].mode`", BRAIN_MODES, &self.brain.mode)); } + problems.extend(self.validate_harnesses()); + problems.extend(self.validate_users()); if !TOOL_PROVIDERS.contains(&self.tools.provider.as_str()) { @@ -557,6 +623,190 @@ impl CompanyManifest { /// filled in under the wrong mode is not a harmless leftover: it is an /// operator who believes they have granted someone access and has not, and /// the symptom is an eligible-looking address that can never sign in. + /// Validates the `[[harness]]` block and every agent's binding to it. + /// + /// A section on the wrong kind is an **error, not an ignored key**, for the + /// same reason a bundle carrying both roster forms is: a silently discarded + /// declaration stays invisible until the thing it configured misbehaves, and + /// "my model setting does nothing" is a very expensive way to learn that + /// `[harness.inference]` needs `kind = "built_in"`. + fn validate_harnesses(&self) -> Vec { + let mut problems = Vec::new(); + + // An absent block is the implicit built_in harness, which is always + // valid — and is what every shipped company has. Nothing to check. + if self.harnesses.is_empty() { + if let Some(agent) = self.agents.iter().find(|a| a.harness.is_some()) { + let named = agent.harness.as_deref().unwrap_or_default(); + problems.push(format!( + "agent `{}` names harness `{named}`, but the manifest declares no `[[harness]]`. \ + Declare it, or drop the `harness` field to use the built-in default.", + agent.id + )); + } + return problems; + } + + let mut seen: BTreeSet<&str> = BTreeSet::new(); + for harness in &self.harnesses { + let id = harness.id.trim(); + if id.is_empty() { + problems.push("`[[harness]]` entries must each set a non-empty `id`.".into()); + } else if !is_snake_case(id) { + problems.push(format!( + "`[[harness]].id` `{id}` is invalid — use snake_case, the same shape as an agent id." + )); + } else if !seen.insert(id) { + problems.push(format!( + "`[[harness]].id` `{id}` is declared more than once — harness ids must be unique." + )); + } + + if !HARNESS_KINDS.contains(&harness.kind.as_str()) { + problems.push(one_of( + &format!("`[[harness]]` `{id}`'s `kind`"), + HARNESS_KINDS, + &harness.kind, + )); + // The per-kind checks below all read `kind`; with an unknown one + // they would report confusing follow-on problems. + continue; + } + + match harness.kind.as_str() { + "built_in" => { + if harness.acp.is_some() { + problems.push(format!( + "`[[harness]]` `{id}` is `kind = \"built_in\"` but declares `[harness.acp]`. \ + An embedded harness has no ACP transport — set `kind = \"acp\"` or drop the section." + )); + } + } + "acp" => { + if harness.inference.is_some() { + problems.push(format!( + "`[[harness]]` `{id}` is `kind = \"acp\"` but declares `[harness.inference]`. \ + An ACP agent runs on its own credential — drop the section, or use `kind = \"built_in\"`." + )); + } + problems.extend(self.validate_acp_harness(id, harness)); + } + _ => unreachable!("kind was checked against HARNESS_KINDS above"), + } + } + + let defaults = self.harnesses.iter().filter(|h| h.default).count(); + if defaults == 0 { + problems.push(format!( + "no `[[harness]]` sets `default = true` — exactly one must, so an agent naming no \ + harness has somewhere to run. Candidates: {}.", + join_backticked( + &self + .harnesses + .iter() + .map(|h| h.id.as_str()) + .collect::>() + ) + )); + } else if defaults > 1 { + problems.push(format!( + "{defaults} `[[harness]]` entries set `default = true` — exactly one must: {}.", + join_backticked( + &self + .harnesses + .iter() + .filter(|h| h.default) + .map(|h| h.id.as_str()) + .collect::>() + ) + )); + } + + for agent in &self.agents { + let Some(named) = agent.harness.as_deref().map(str::trim) else { + continue; + }; + if !seen.contains(named) { + problems.push(format!( + "agent `{}` names harness `{named}`, which no `[[harness]]` declares. Declared: {}.", + agent.id, + join_backticked(&seen.iter().copied().collect::>()) + )); + } + } + + problems + } + + /// The `[harness.acp]` cross-field rules: each transport requires its own + /// addressing field and forbids the other's, so a manifest cannot claim to + /// spawn a local agent *and* name a remote runner. + fn validate_acp_harness(&self, id: &str, harness: &Harness) -> Vec { + let mut problems = Vec::new(); + let Some(acp) = harness.acp.as_ref() else { + problems.push(format!( + "`[[harness]]` `{id}` is `kind = \"acp\"` but declares no `[harness.acp]` — \ + it needs a `transport`." + )); + return problems; + }; + + if !ACP_TRANSPORTS.contains(&acp.transport.as_str()) { + problems.push(one_of( + &format!("`[[harness]]` `{id}`'s `[harness.acp].transport`"), + ACP_TRANSPORTS, + &acp.transport, + )); + return problems; + } + + match acp.transport.as_str() { + "local" => { + match acp.agent.as_deref() { + None => problems.push(format!( + "`[[harness]]` `{id}` uses `transport = \"local\"` but names no `agent` — \ + one of {}.", + join_backticked(ACP_AGENTS) + )), + Some(agent) if !ACP_AGENTS.contains(&agent) => problems.push(one_of( + &format!("`[[harness]]` `{id}`'s `[harness.acp].agent`"), + ACP_AGENTS, + agent, + )), + Some(_) => {} + } + if acp.runner.is_some() { + problems.push(format!( + "`[[harness]]` `{id}` uses `transport = \"local\"` but names a `runner`. \ + A local agent is spawned on this machine — use `transport = \"runner\"` to reach one elsewhere." + )); + } + } + "runner" => { + if acp + .runner + .as_deref() + .map(str::trim) + .unwrap_or_default() + .is_empty() + { + problems.push(format!( + "`[[harness]]` `{id}` uses `transport = \"runner\"` but names no `runner`." + )); + } + if acp.agent.is_some() { + problems.push(format!( + "`[[harness]]` `{id}` uses `transport = \"runner\"` but names an `agent`. \ + A runner advertises the harnesses it can drive — this host does not choose one for it." + )); + } + } + _ => unreachable!("transport was checked against ACP_TRANSPORTS above"), + } + + problems + } + fn validate_users(&self) -> Vec { let mut problems = Vec::new(); let mode = self.users.mode.as_str(); @@ -627,6 +877,19 @@ impl CompanyManifest { let _ = writeln!(out, "You own: {role}"); } let _ = writeln!(out, "Brain: {}", self.brain.mode); + // Always the effective set, so a company with no `[[harness]]` block + // prints the implicit harness it actually runs on rather than nothing. + let default_harness = self.default_harness_id(); + let harnesses = self + .effective_harnesses() + .iter() + .map(|h| { + let marker = if h.id == default_harness { "*" } else { "" }; + format!("{}{marker} ({})", h.id, h.kind) + }) + .collect::>() + .join(", "); + let _ = writeln!(out, "Harness: {harnesses}"); let _ = writeln!(out, "Policy: {}", self.policy.mode); let _ = writeln!(out, "Tools: {}", self.tools.provider); if let Some(monthly) = self.budget.monthly_usd { @@ -1690,3 +1953,314 @@ mod tests { std::fs::remove_dir_all(&dir).ok(); } } + +#[cfg(test)] +mod harness_tests { + use super::*; + + fn parse(text: &str) -> CompanyManifest { + toml::from_str(text).expect("valid toml") + } + + /// Every problem mentioning `harness`, so a test asserting on this block is + /// not perturbed by unrelated validation output. + fn harness_problems(m: &CompanyManifest) -> Vec { + m.validate() + .into_iter() + .filter(|p| p.contains("harness")) + .collect() + } + + const BASE: &str = "[company]\nname = \"X\"\n\n[[agent]]\nid = \"ceo\"\nrole = \"CEO\"\n"; + + /// The compatibility case, and the one every shipped company under + /// `companies/` hits: no `[[harness]]` block at all still yields exactly one + /// harness — `built_in`, default, on the company-level `[inference]`. + /// + /// This is the test that makes "named harnesses" a purely additive feature. + #[test] + fn a_manifest_with_no_harness_block_gets_one_implicit_built_in_default() { + let manifest = parse(BASE); + + assert!(manifest.harnesses.is_empty(), "nothing was declared"); + + let effective = manifest.effective_harnesses(); + assert_eq!(effective.len(), 1); + assert_eq!(effective[0].id, IMPLICIT_HARNESS_ID); + assert_eq!(effective[0].kind, "built_in"); + assert!(effective[0].default); + assert!(effective[0].inference.is_none(), "inherits `[inference]`"); + + assert_eq!(manifest.default_harness_id(), IMPLICIT_HARNESS_ID); + assert_eq!( + manifest.harness_for("ceo").map(|h| h.id), + Some(IMPLICIT_HARNESS_ID.to_string()), + "an agent naming no harness lands on the implicit one" + ); + assert!(harness_problems(&manifest).is_empty()); + } + + /// Naming a harness when none is declared is an error rather than a silent + /// fallback to the implicit one: the operator wrote down an intent, and + /// quietly ignoring it is how "my agent is on the wrong model" happens. + #[test] + fn naming_a_harness_with_no_harness_block_is_rejected() { + let manifest = parse(&format!("{BASE}harness = \"deep\"\n")); + let problems = harness_problems(&manifest); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!(problems[0].contains("ceo") && problems[0].contains("deep")); + } + + #[test] + fn agents_route_to_their_named_harness_and_others_to_the_default() { + let manifest = parse( + r#" +[company] +name = "X" + +[[agent]] +id = "ceo" +role = "CEO" + +[[agent]] +id = "researcher" +role = "Researcher" +harness = "deep" + +[[harness]] +id = "embedded" +kind = "built_in" +default = true + +[[harness]] +id = "deep" +kind = "built_in" + +[harness.inference] +provider = "openrouter" +"#, + ); + assert!(harness_problems(&manifest).is_empty()); + assert_eq!(manifest.default_harness_id(), "embedded"); + assert_eq!( + manifest.harness_for("ceo").map(|h| h.id), + Some("embedded".to_string()) + ); + assert_eq!( + manifest.harness_for("researcher").map(|h| h.id), + Some("deep".to_string()) + ); + // The sub-table attached to the *second* entry, not the first — the + // array-of-tables shape that is easy to misread. + let deep = manifest.harness_for("researcher").expect("declared"); + assert_eq!( + deep.inference.as_ref().and_then(|i| i.provider.clone()), + Some("openrouter".to_string()) + ); + assert!( + manifest + .effective_harnesses() + .iter() + .find(|h| h.id == "embedded") + .expect("declared") + .inference + .is_none() + ); + } + + #[test] + fn an_agent_naming_an_undeclared_harness_is_rejected() { + let manifest = parse(&format!( + "{BASE}harness = \"ghost\"\n\n[[harness]]\nid = \"embedded\"\nkind = \"built_in\"\ndefault = true\n" + )); + let problems = harness_problems(&manifest); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!(problems[0].contains("ghost") && problems[0].contains("ceo")); + assert!( + problems[0].contains("embedded"), + "names what IS declared: {}", + problems[0] + ); + } + + #[test] + fn duplicate_harness_ids_are_rejected() { + let manifest = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"built_in\"\ndefault = true\n\n[[harness]]\nid = \"a\"\nkind = \"built_in\"\n" + )); + let problems = harness_problems(&manifest); + assert!( + problems.iter().any(|p| p.contains("more than once")), + "{problems:?}" + ); + } + + /// Zero and two defaults are both errors. Zero would leave an agent naming + /// no harness with nowhere to run; two makes the answer depend on list + /// order, which is exactly what marking a default exists to avoid. + #[test] + fn there_must_be_exactly_one_default_harness() { + let none = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"built_in\"\n\n[[harness]]\nid = \"b\"\nkind = \"built_in\"\n" + )); + let problems = harness_problems(&none); + assert!( + problems.iter().any(|p| p.contains("no `[[harness]]` sets")), + "{problems:?}" + ); + assert!( + problems + .iter() + .any(|p| p.contains("`a`") && p.contains("`b`")), + "names the candidates: {problems:?}" + ); + + let two = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"built_in\"\ndefault = true\n\n[[harness]]\nid = \"b\"\nkind = \"built_in\"\ndefault = true\n" + )); + let problems = harness_problems(&two); + assert!( + problems.iter().any(|p| p.contains("2 `[[harness]]`")), + "{problems:?}" + ); + } + + /// A section on the wrong kind is an error, not an ignored key — both + /// directions. + #[test] + fn a_section_on_the_wrong_kind_is_an_error_not_an_ignored_key() { + let inference_on_acp = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"acp\"\ndefault = true\n\n[harness.acp]\ntransport = \"local\"\nagent = \"claude\"\n\n[harness.inference]\nprovider = \"openrouter\"\n" + )); + let problems = harness_problems(&inference_on_acp); + assert!( + problems + .iter() + .any(|p| p.contains("[harness.inference]") && p.contains("own credential")), + "{problems:?}" + ); + + let acp_on_built_in = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"built_in\"\ndefault = true\n\n[harness.acp]\ntransport = \"local\"\nagent = \"claude\"\n" + )); + let problems = harness_problems(&acp_on_built_in); + assert!( + problems.iter().any(|p| p.contains("[harness.acp]")), + "{problems:?}" + ); + } + + #[test] + fn an_unknown_harness_kind_is_rejected_without_confusing_follow_on_problems() { + let manifest = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"telepathy\"\ndefault = true\n" + )); + let problems = harness_problems(&manifest); + assert_eq!( + problems.len(), + 1, + "one problem, not a cascade: {problems:?}" + ); + assert!(problems[0].contains("telepathy") && problems[0].contains("built_in")); + } + + /// Each ACP transport requires its own addressing field and forbids the + /// other's, so a manifest cannot claim to spawn a local agent *and* name a + /// remote runner. + #[test] + fn acp_transports_require_their_own_addressing_field() { + let cases: &[(&str, &str)] = &[ + ("transport = \"local\"\n", "names no `agent`"), + ( + "transport = \"local\"\nagent = \"claude\"\nrunner = \"laptop\"\n", + "but names a `runner`", + ), + ("transport = \"runner\"\n", "names no `runner`"), + ( + "transport = \"runner\"\nrunner = \"laptop\"\nagent = \"claude\"\n", + "but names an `agent`", + ), + ("transport = \"carrier_pigeon\"\n", "must be one of"), + ( + "transport = \"local\"\nagent = \"emacs\"\n", + "must be one of", + ), + ]; + for (acp, expected) in cases { + let manifest = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"acp\"\ndefault = true\n\n[harness.acp]\n{acp}" + )); + let problems = harness_problems(&manifest); + assert!( + problems.iter().any(|p| p.contains(expected)), + "`{acp}` should report {expected:?}, got {problems:?}" + ); + } + } + + #[test] + fn a_valid_acp_harness_of_each_transport_passes() { + for acp in [ + "transport = \"local\"\nagent = \"claude\"\n", + "transport = \"runner\"\nrunner = \"stevens_laptop\"\n", + ] { + let manifest = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"acp\"\ndefault = true\n\n[harness.acp]\n{acp}" + )); + assert!( + harness_problems(&manifest).is_empty(), + "`{acp}` should be valid: {:?}", + harness_problems(&manifest) + ); + } + } + + #[test] + fn an_acp_harness_with_no_acp_section_is_rejected() { + let manifest = parse(&format!( + "{BASE}\n[[harness]]\nid = \"a\"\nkind = \"acp\"\ndefault = true\n" + )); + let problems = harness_problems(&manifest); + assert!( + problems.iter().any(|p| p.contains("needs a `transport`")), + "{problems:?}" + ); + } + + #[test] + fn harness_ids_must_be_snake_case() { + let manifest = parse(&format!( + "{BASE}\n[[harness]]\nid = \"My Harness\"\nkind = \"built_in\"\ndefault = true\n" + )); + let problems = harness_problems(&manifest); + assert!( + problems.iter().any(|p| p.contains("snake_case")), + "{problems:?}" + ); + } + + /// The per-file roster form carries `harness` through, so the two authoring + /// forms agree. + #[test] + fn a_per_file_agent_carries_its_harness_binding() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write( + dir.path().join(MANIFEST_FILE), + "[company]\nname = \"X\"\n\n[[harness]]\nid = \"embedded\"\nkind = \"built_in\"\ndefault = true\n\n[[harness]]\nid = \"deep\"\nkind = \"built_in\"\n", + ) + .expect("write manifest"); + let agents = dir.path().join(super::super::agent_file::AGENTS_DIR); + std::fs::create_dir_all(&agents).expect("agents dir"); + std::fs::write( + agents.join("researcher.toml"), + "role = \"Researcher\"\nharness = \"deep\"\n", + ) + .expect("write agent"); + + let manifest = CompanyManifest::from_path(dir.path()).expect("parses"); + assert_eq!( + manifest.harness_for("researcher").map(|h| h.id), + Some("deep".to_string()) + ); + } +} diff --git a/src/company/mod.rs b/src/company/mod.rs index fe12b63fc..29ac6a4f5 100644 --- a/src/company/mod.rs +++ b/src/company/mod.rs @@ -125,16 +125,18 @@ pub(crate) use manifest::is_snake_case; pub use manifest::{DELEGATES_TO_WILDCARD, LEGACY_MANIFEST_FILE, Located, MANIFEST_FILE, discover}; pub use skill_file::{SkillDoc, load_dir_skills, parse_skill_md, render_skill_md}; pub use types::{ - Agent, BRAIN_MODES, Brain, Budget, ChannelConfig, Company, CompanyManifest, ComposioTools, - Connection, ContextAccess, ContextEntry, DEFAULT_ALWAYS_APPROVE, DEFAULT_MAX_DELEGATION_DEPTH, + ACP_AGENTS, ACP_TRANSPORTS, AcpHarness, Agent, BRAIN_MODES, Brain, Budget, ChannelConfig, + Company, CompanyManifest, ComposioTools, Connection, ContextAccess, ContextEntry, + DEFAULT_ALWAYS_APPROVE, DEFAULT_HARNESS_KIND, DEFAULT_MAX_DELEGATION_DEPTH, DEFAULT_MAX_IN_FLIGHT_RUNS, DEFAULT_SEARCH_DAILY_CALLS, GATEABLE_NAMESPACES, GroupChat, - INFERENCE_PROVIDERS, INFERENCE_TIERS, Inference, KNOWN_CHANNELS, LedgerAccess, LedgerGrant, - MAX_DELEGATION_DEPTH_BOUNDS, McpServer, ORCHESTRATOR_TIER, PLAN_NAMES, PLAN_PERIODS, - POLICY_MODES, PROMPT_CLASSES, PROMPT_FILE_BUDGET_CHARS, PROVISIONED_POLICY_MODE, Place, Plan, - Policy, Schedule, Skill, TIERS, TOOL_PROVIDERS, Tools, grants_chargebee_explicit, - grants_composio_explicit, grants_hosting_explicit, grants_media_explicit, - grants_paypal_explicit, grants_repo_explicit, grants_repo_write_explicit, - grants_search_explicit, grants_workspace_write_explicit, orchestrator_id, + HARNESS_KINDS, Harness, IMPLICIT_HARNESS_ID, INFERENCE_PROVIDERS, INFERENCE_TIERS, Inference, + KNOWN_CHANNELS, LedgerAccess, LedgerGrant, MAX_DELEGATION_DEPTH_BOUNDS, McpServer, + ORCHESTRATOR_TIER, PLAN_NAMES, PLAN_PERIODS, POLICY_MODES, PROMPT_CLASSES, + PROMPT_FILE_BUDGET_CHARS, PROVISIONED_POLICY_MODE, Place, Plan, Policy, Schedule, Skill, TIERS, + TOOL_PROVIDERS, Tools, grants_chargebee_explicit, grants_composio_explicit, + grants_hosting_explicit, grants_media_explicit, grants_paypal_explicit, grants_repo_explicit, + grants_repo_write_explicit, grants_search_explicit, grants_workspace_write_explicit, + orchestrator_id, }; pub use workflow_file::{ STAGELESS_SCHEDULE_REFUSAL, STAGELESS_WORKFLOW_NOTICE, UNDELIVERABLE_SCHEDULE_REFUSAL, diff --git a/src/company/prompt.rs b/src/company/prompt.rs index fa0150935..22c20875c 100644 --- a/src/company/prompt.rs +++ b/src/company/prompt.rs @@ -193,6 +193,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: Vec::new(), delegates_to: Vec::new(), context: None, diff --git a/src/company/types.rs b/src/company/types.rs index 9d905f86b..1c83c9938 100644 --- a/src/company/types.rs +++ b/src/company/types.rs @@ -28,14 +28,51 @@ pub const AUTH_MODES: &[&str] = &["email", "wallet", "none"]; /// Inference providers selectable in `[inference].provider` (issue #56 — BYOK). /// -/// * `managed` — the hosted TinyHumans / Medulla brain (the default path). -/// * `openrouter` — OpenRouter's OpenAI-compatible aggregator (needs a key + -/// the `HTTP-Referer` / `X-Title` attribution headers). +/// * `openrouter` — OpenRouter's OpenAI-compatible aggregator, and the default. +/// With **no** key it resolves to the platform endpoint and the subscription +/// pays; with a tenant `sk-or-…` it goes direct to OpenRouter on the tenant's +/// own account. Either way it carries OpenRouter's `HTTP-Referer` / `X-Title` +/// attribution headers. /// * `openai_compatible` — any OpenAI-compatible endpoint the tenant runs /// (needs a `base_url`, usually a key). /// * `ollama` — a local Ollama server's OpenAI-compatible surface (needs a /// `base_url`; no key). -pub const INFERENCE_PROVIDERS: &[&str] = &["managed", "openrouter", "openai_compatible", "ollama"]; +/// +/// `managed` was removed: OpenCompany no longer exposes its own model SKUs, so +/// there is nothing for a distinct managed kind to name. A manifest or stored +/// runtime blob still saying `managed` aliases to `openrouter` — see +/// [`inference::LEGACY_MANAGED`](crate::company::inference::LEGACY_MANAGED). +pub const INFERENCE_PROVIDERS: &[&str] = &["openrouter", "openai_compatible", "ollama"]; + +/// Harness kinds selectable in `[[harness]].kind`. +/// +/// * `built_in` — the embedded OpenHuman loop, in this process, against the +/// inference provider the harness itself declares. +/// * `acp` — an external agent driven over the Agent Client Protocol. Runs on +/// whatever credential *it* holds, so it needs no `[harness.inference]`. +pub const HARNESS_KINDS: &[&str] = &["built_in", "acp"]; + +/// The default harness kind, used for the implicit harness a company with no +/// `[[harness]]` block gets. +pub const DEFAULT_HARNESS_KIND: &str = "built_in"; + +/// The id given to the implicit harness synthesized for a company that declares +/// no `[[harness]]` block. +pub const IMPLICIT_HARNESS_ID: &str = "default"; + +/// Transports selectable in `[harness.acp].transport`. +/// +/// A remote runner is a *transport*, not a third harness kind: `RunnerDispatch` +/// already implements the same `AcpAgent` port the local subprocess does, so the +/// only thing that differs is how bytes reach the agent. +pub const ACP_TRANSPORTS: &[&str] = &["local", "runner"]; + +/// ACP agents selectable in `[harness.acp].agent`, for `transport = "local"`. +/// +/// Kept in step with the desktop's own `ACP_HARNESSES` catalogue, which encodes +/// how to put each one into ACP mode — guessing those arguments wrong spawns a +/// process that hangs waiting for interactive input. +pub const ACP_AGENTS: &[&str] = &["claude", "codex", "goose"]; /// The abstract cognition tiers the tenant `[inference].models` table maps to /// concrete provider model ids. These are the workload names the harness @@ -363,6 +400,15 @@ pub struct CompanyManifest { /// Brain selection. #[serde(default)] pub brain: Brain, + /// The named execution engines this company's agents run on. Renamed from + /// the `[[harness]]` array-of-tables. + /// + /// Empty (the default) means one implicit `built_in` harness on the + /// company-level [`inference`](Self::inference) — resolve through + /// [`effective_harnesses`](Self::effective_harnesses) rather than reading + /// this field, so the implicit case is never forgotten. + #[serde(default, rename = "harness")] + pub harnesses: Vec, /// Per-tenant Bring-Your-Own-Key inference routing (issue #56). Declarative /// intent — a provider kind, an OpenAI-compatible `base_url`, an optional /// *named* secret key (`api_key_secret`), and an abstract-tier → model map. @@ -470,6 +516,15 @@ pub struct Agent { /// Cognition tier hint; never selects a model. #[serde(default)] pub tier: Option, + /// Which `[[harness]]` this agent runs its turns on, by id. + /// + /// `None` means the harness marked `default = true`. Deliberately separate + /// from [`tier`](Self::tier): a tier names a *workload* and is resolved + /// against whatever provider the harness turns out to use, whereas this + /// picks the engine and the credential. An agent can keep its tier while + /// moving between harnesses. + #[serde(default)] + pub harness: Option, /// Tool grant globs, intersected with `[tools].allow`. #[serde(default)] pub tools: Vec, @@ -1031,6 +1086,92 @@ fn default_brain_mode() -> String { "hosted".to_string() } +/// A `[[harness]]` entry — one named execution engine the company's agents may +/// run their turns on. +/// +/// A company declares a set of these and binds each agent to one with +/// [`Agent::harness`], so a single roster can span a cheap model, an expensive +/// one, and the operator's own Claude Code — the last needing no credential from +/// us at all. +/// +/// Like [`McpServer`] and [`Inference`], this is declarative intent and **never** +/// carries a token: a `built_in` harness's credential is named by +/// `[harness.inference].api_key_secret`, and an `acp` harness holds its own. +/// +/// The TOML shape is an array-of-tables with sub-tables: +/// +/// ```toml +/// [[harness]] +/// id = "embedded" +/// kind = "built_in" +/// default = true +/// +/// [harness.inference] # attaches to the entry above +/// provider = "openrouter" +/// ``` +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct Harness { + /// snake_case, unique within the company. Agents name this. + pub id: String, + /// One of [`HARNESS_KINDS`]. + #[serde(default = "default_harness_kind")] + pub kind: String, + /// Whether agents naming no harness run here. Exactly one entry must set + /// it, whenever any `[[harness]]` is declared. + #[serde(default)] + pub default: bool, + /// This harness's own inference routing. `built_in` only. Absent falls back + /// to the company-level `[inference]` section. + #[serde(default)] + pub inference: Option, + /// How to reach the external agent. `acp` only. + #[serde(default)] + pub acp: Option, +} + +fn default_harness_kind() -> String { + DEFAULT_HARNESS_KIND.to_string() +} + +impl Harness { + /// The implicit harness a company with no `[[harness]]` block runs on: one + /// `built_in` entry, marked default, inheriting the company-level + /// `[inference]`. + /// + /// Synthesized rather than required in every manifest so that adding named + /// harnesses changes nothing for a company that never asked for them — + /// every bundle under `companies/` lands here. + pub fn implicit() -> Self { + Self { + id: IMPLICIT_HARNESS_ID.to_string(), + kind: DEFAULT_HARNESS_KIND.to_string(), + default: true, + inference: None, + acp: None, + } + } + + /// Whether this is the embedded loop — the only kind that consults + /// `[inference]`. + pub fn is_built_in(&self) -> bool { + self.kind == "built_in" + } +} + +/// `[harness.acp]` — how to reach an external ACP agent. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct AcpHarness { + /// One of [`ACP_TRANSPORTS`]. + #[serde(default)] + pub transport: String, + /// Which agent to spawn, one of [`ACP_AGENTS`]. `local` transport only. + #[serde(default)] + pub agent: Option, + /// Which registered runner holds this scope. `runner` transport only. + #[serde(default)] + pub runner: Option, +} + /// `[inference]` — per-tenant Bring-Your-Own-Key inference routing (issue #56). /// /// This is declarative intent, shaped like [`McpServer`]: it names a provider diff --git a/src/harness/acp/mod.rs b/src/harness/acp/mod.rs new file mode 100644 index 000000000..b3523375f --- /dev/null +++ b/src/harness/acp/mod.rs @@ -0,0 +1,16 @@ +//! The ACP harness: a company turn served by an **external** agent over the +//! Agent Client Protocol. +//! +//! Gated behind the `acp` feature because nothing in a default build can reach +//! it — the endpoint that would drive it lives behind the same feature, and +//! `/acp` is a reserved prefix that 404s without it. Compiling it +//! unconditionally meant a surface that no lane ran and no route served (issue +//! #475). +//! +//! The transport is deliberately *not* here: a subprocess over stdio for the +//! desktop and a WebSocket for a runner belong to their own crates, so +//! [`run_turn`] defines an [`AcpAgent`](run_turn::AcpAgent) port and folds +//! whatever it reports. The same inversion the storage ports use. + +#[cfg(feature = "acp")] +pub mod run_turn; diff --git a/src/harness/acp_run_turn.rs b/src/harness/acp/run_turn.rs similarity index 63% rename from src/harness/acp_run_turn.rs rename to src/harness/acp/run_turn.rs index f68c6675e..08a186a19 100644 --- a/src/harness/acp_run_turn.rs +++ b/src/harness/acp/run_turn.rs @@ -43,10 +43,12 @@ //! no tool call produced. use std::sync::Arc; +use std::time::Duration; use async_trait::async_trait; use crate::Result; +use crate::error::OpenCompanyError; use crate::harness::TurnOutcome; use crate::ports::types::{CompanyId, TurnStep, TurnStepKind, TurnStepStatus}; use crate::runtime::delegation::RunTurn; @@ -251,19 +253,61 @@ impl RunTurn for AcpRunTurn { } impl AcpRunTurn { + /// How long a cancelled turn may keep running before the waiter gives up. + /// + /// Cancellation in ACP is cooperative: `session/cancel` is a notification, + /// and a harness inside a long tool call only notices when that call + /// returns. So the post-cancel wait stays, but it is bounded — a cancelled + /// turn that has not drained its output within this window is abandoned, + /// not waited on forever. The window is generous enough for a slow tool + /// call to finish and its updates to flush. + const CANCEL_GRACE: Duration = Duration::from_secs(30); + + /// Bound on a single `session/cancel` round trip. A cancel that never + /// answers — a wedged host, a dead subprocess — must not pin the steered + /// turn forever; the grace wait is what actually reaps a turn that ignores + /// the cancel, and this bound just keeps the attempt to tell it from + /// blocking that. + const CANCEL_RPC_TIMEOUT: Duration = Duration::from_secs(5); + /// A turn that can be cancelled while it runs. /// /// The turn and the steer check race each other. A cancel forwards /// `session/cancel` and then **keeps waiting** rather than abandoning the /// turn: ACP cancellation is cooperative, the agent still answers with /// `stopReason: "cancelled"`, and dropping the future here would leave a - /// harness mid-tool-call with nothing reading its output. + /// harness mid-tool-call with nothing reading its output. That wait is + /// bounded by [`Self::CANCEL_GRACE`]: a turn that ignores the cancel past + /// the grace window is abandoned with an error, not awaited forever. async fn steered( &self, company: &CompanyId, agent_id: &str, message: &str, control: &crate::company::steer::SteerControl, + ) -> Result { + self.steered_with_grace( + company, + agent_id, + message, + control, + Self::CANCEL_GRACE, + Self::CANCEL_RPC_TIMEOUT, + ) + .await + } + + /// [`Self::steered`] with both timing bounds made explicit — the post-cancel + /// grace and the per-cancel-RPC bound — so the tests can expire them in + /// milliseconds rather than waiting out the real windows. + async fn steered_with_grace( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + control: &crate::company::steer::SteerControl, + grace: Duration, + cancel_rpc: Duration, ) -> Result { let key = Self::session_key(company, agent_id); let turn = self.agent.prompt(company, &key, message); @@ -277,10 +321,49 @@ impl AcpRunTurn { // reads the action to decide what happens to the card, and // consuming it here would leave it with nothing to read. if control.pending().is_some() { - // Advisory. Told, then waited for — see above. - let _ = self.agent.cancel(company, &key).await; - let outcome = (&mut turn).await?; - return Ok(fold(outcome)); + // Advisory. Told, then waited for — see above. The RPC + // itself is bounded so a cancel that never answers (a + // wedged host, a dead subprocess) cannot block the turn; + // both outcomes below are logged and the flow continues. + match tokio::time::timeout(cancel_rpc, self.agent.cancel(company, &key)) + .await + { + Ok(Ok(())) => {} + Ok(Err(err)) => { + tracing::warn!(%err, "[harness::acp] cancel failed for session {key}"); + } + Err(_elapsed) => { + tracing::warn!("[harness::acp] cancel timed out for session {key}"); + } + } + match tokio::time::timeout(grace, &mut turn).await { + Ok(outcome) => return Ok(fold(outcome?)), + Err(_elapsed) => { + // The agent ignored the cancel past the grace + // window. The port has no abort/reset seam — + // `cancel` is all there is — so the best this + // side can do is nudge once more and drop the + // turn. Dropping the future ends the reader on + // this session; the agent's own `session/cancel` + // handling (or the host reaping the subprocess) + // is the recovery path for the work it still + // holds. A later turn on the same key opens a + // fresh `session/prompt`, which the agent treats + // as a new turn rather than an overlap. The + // nudge is bounded the same way: it is best + // effort, and the abandonment is the point. + let _ = tokio::time::timeout( + cancel_rpc, + self.agent.cancel(company, &key), + ) + .await; + return Err(OpenCompanyError::Harness(format!( + "the agent did not stop within {}s of a cancel; \ + abandoning the turn", + grace.as_secs() + ))); + } + } } } } @@ -440,19 +523,65 @@ mod test { } /// An agent that answers from a script, so the trait impl can be driven. + /// + /// `hang` makes `prompt` never resolve (the grace-expiry path) and + /// `cancel_fails` makes `cancel` error (the logged-failure path). `cancels` + /// counts cancel calls so a test can assert the grace path nudged twice. + /// + /// `hold_for_cancel` makes `prompt` wait until the first `cancel` arrives — + /// the shape of a turn that is mid-tool-call when the operator steers, which + /// is exactly the window the advisory cancel exists for. Without the gate a + /// prompt that resolves immediately exits the loop before the steer check + /// ever runs, and the cancel path goes unexercised. `cancel_hangs` makes + /// `cancel` never answer (the bounded-RPC path). struct Scripted { turn: AcpTurn, - cancelled: std::sync::Arc, + hang: bool, + hold_for_cancel: bool, + cancel_hangs: bool, + cancel_fails: bool, + cancels: std::sync::Arc, + cancel_started: tokio::sync::Notify, + } + + impl Scripted { + fn answering(updates: Vec) -> Self { + Self { + turn: AcpTurn { + updates, + stop_reason: "end_turn".into(), + }, + hang: false, + hold_for_cancel: false, + cancel_hangs: false, + cancel_fails: false, + cancels: Default::default(), + cancel_started: tokio::sync::Notify::new(), + } + } } #[async_trait] impl AcpAgent for Scripted { async fn prompt(&self, _c: &CompanyId, _k: &str, _m: &str) -> Result { + if self.hang { + std::future::pending::<()>().await; + } + if self.hold_for_cancel { + self.cancel_started.notified().await; + } Ok(self.turn.clone()) } async fn cancel(&self, _c: &CompanyId, _k: &str) -> Result<()> { - self.cancelled - .store(true, std::sync::atomic::Ordering::SeqCst); + self.cancels + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.cancel_started.notify_waiters(); + if self.cancel_hangs { + std::future::pending::<()>().await; + } + if self.cancel_fails { + return Err(OpenCompanyError::Harness("cancel rejected".into())); + } Ok(()) } } @@ -466,25 +595,19 @@ mod test { /// object-safe would compile here and fail at the one site that matters. #[tokio::test] async fn it_is_usable_through_the_run_turn_seam() { - let agent = Arc::new(Scripted { - turn: AcpTurn { - updates: vec![ - AcpUpdate::ThoughtChunk, - AcpUpdate::ToolCall { - id: "t1".into(), - title: "Read".into(), - }, - AcpUpdate::ToolCallUpdate { - id: "t1".into(), - status: "completed".into(), - result: Some("4 items".into()), - }, - AcpUpdate::MessageChunk("all done".into()), - ], - stop_reason: "end_turn".into(), + let agent = Arc::new(Scripted::answering(vec![ + AcpUpdate::ThoughtChunk, + AcpUpdate::ToolCall { + id: "t1".into(), + title: "Read".into(), }, - cancelled: Default::default(), - }); + AcpUpdate::ToolCallUpdate { + id: "t1".into(), + status: "completed".into(), + result: Some("4 items".into()), + }, + AcpUpdate::MessageChunk("all done".into()), + ])); let run_turn: &dyn RunTurn = &AcpRunTurn::new(agent); let outcome = run_turn @@ -504,13 +627,9 @@ mod test { // `stopReason: "cancelled"`. Abandoning the future on a steer would // leave a harness mid-tool-call with nothing reading its output, so the // contract is that a steered turn still produces an outcome. - let agent = Arc::new(Scripted { - turn: AcpTurn { - updates: vec![AcpUpdate::MessageChunk("partial".into())], - stop_reason: "cancelled".into(), - }, - cancelled: Default::default(), - }); + let agent = Arc::new(Scripted::answering(vec![AcpUpdate::MessageChunk( + "partial".into(), + )])); let run_turn: &dyn RunTurn = &AcpRunTurn::new(agent); let control = crate::company::steer::SteerControl::new(); control.request(crate::company::steer::SteerAction::Cancel); @@ -528,6 +647,111 @@ mod test { ); } + #[tokio::test] + async fn a_failed_cancel_is_logged_and_the_turn_still_drains() { + // `session/cancel` can fail (the subprocess is mid-shutdown, say), but + // that must not turn a cancelled turn into a failure of its own: the + // cancel is advisory, the error is logged, and the turn still answers. + // The prompt holds until the cancel arrives so the steer check is + // actually reached — a prompt that resolves first would exit the loop + // and leave the cancel path unexercised. + let mut agent = Scripted::answering(vec![AcpUpdate::MessageChunk("done".into())]); + agent.cancel_fails = true; + agent.hold_for_cancel = true; + let cancels = agent.cancels.clone(); + let agent = Arc::new(agent); + let run_turn: &dyn RunTurn = &AcpRunTurn::new(agent); + let control = crate::company::steer::SteerControl::new(); + control.request(crate::company::steer::SteerAction::Cancel); + + let outcome = run_turn + .run_steered(&CompanyId::new("acme"), "ceo", "go", &control, None, None) + .await + .expect("a failed cancel still ends in a turn"); + assert_eq!(outcome.reply, "done"); + assert_eq!( + cancels.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the failed cancel was still attempted exactly once" + ); + } + + #[tokio::test] + async fn a_hung_cancel_rpc_does_not_block_the_turn() { + // A cancellation RPC that never answers — a wedged host, a dead + // subprocess — must not pin the steered turn forever. Both cancel calls + // are bounded, so the turn still settles on the grace schedule. + let mut agent = Scripted::answering(vec![AcpUpdate::MessageChunk("done".into())]); + agent.cancel_hangs = true; + agent.hold_for_cancel = true; + let agent = Arc::new(agent); + let run_turn = AcpRunTurn::new(agent); + let control = crate::company::steer::SteerControl::new(); + control.request(crate::company::steer::SteerAction::Cancel); + + let outcome = tokio::time::timeout( + Duration::from_secs(5), + run_turn.steered_with_grace( + &CompanyId::new("acme"), + "ceo", + "go", + &control, + Duration::from_millis(20), // post-cancel grace + Duration::from_millis(50), // cancel RPC bound + ), + ) + .await + .expect("the turn settles despite a hung cancel RPC") + .expect("the release of the prompt lets the turn answer"); + + assert_eq!(outcome.reply, "done"); + } + + #[tokio::test] + async fn a_cancelled_turn_that_ignores_the_cancel_is_abandoned() { + // A harness inside a tool call that never returns is the one case the + // cooperative wait must not honour: past the grace window the waiter + // drops the turn with an error, and nudges `cancel` once more on the + // way out — the only drain lever the port exposes. + let agent = Arc::new(Scripted { + turn: AcpTurn { + updates: vec![], + stop_reason: "end_turn".into(), + }, + hang: true, + hold_for_cancel: false, + cancel_hangs: false, + cancel_fails: false, + cancels: Default::default(), + cancel_started: tokio::sync::Notify::new(), + }); + let cancels = agent.cancels.clone(); + let run_turn = AcpRunTurn::new(agent); + let control = crate::company::steer::SteerControl::new(); + control.request(crate::company::steer::SteerAction::Cancel); + + let err = run_turn + .steered_with_grace( + &CompanyId::new("acme"), + "ceo", + "go", + &control, + Duration::from_millis(20), + Duration::from_millis(50), + ) + .await + .expect_err("a hung turn is abandoned, not awaited"); + assert!( + format!("{err}").contains("abandoning the turn"), + "the error names the abandonment: {err}" + ); + assert_eq!( + cancels.load(std::sync::atomic::Ordering::SeqCst), + 2, + "one cancel on the steer, one best-effort nudge on the way out" + ); + } + #[test] fn a_session_key_separates_agents_and_companies() { // Two desks sharing a session would share a conversation, and one diff --git a/src/harness/audit.rs b/src/harness/built_in/audit.rs similarity index 100% rename from src/harness/audit.rs rename to src/harness/built_in/audit.rs diff --git a/src/harness/audit/test.rs b/src/harness/built_in/audit/test.rs similarity index 100% rename from src/harness/audit/test.rs rename to src/harness/built_in/audit/test.rs diff --git a/src/harness/brain.rs b/src/harness/built_in/brain.rs similarity index 97% rename from src/harness/brain.rs rename to src/harness/built_in/brain.rs index 72d7e6fc8..a582c4e32 100644 --- a/src/harness/brain.rs +++ b/src/harness/built_in/brain.rs @@ -181,7 +181,19 @@ impl Drop for CheckoutJanitor { /// A [`Brain`] that answers with a live openhuman agent turn. pub struct HarnessBrain { pool: Arc, - deps: HarnessDeps, + deps: Arc, + /// Every harness lane beyond the default, by id, and which agents are bound + /// to which. Empty for a company that declares no `[[harness]]` block — + /// which is every company that has not asked for one — and in that case + /// [`run_turn`](Self::run_turn) hands back the default lane directly, so the + /// single-harness path stays exactly what it was. + lanes: Vec<(String, Arc)>, + /// Agent id -> harness id, for agents bound to a named harness. + bindings: std::collections::HashMap, + /// Declared harnesses this host cannot run, and why. + unavailable: Vec<(String, String)>, + /// The harness id agents naming none run on. + default_harness: String, /// The LLM triage escalation, built on first use (issue #678). /// /// Lazy because it needs the company id, and a brain outlives any one @@ -290,9 +302,20 @@ impl HarnessBrain { /// once and reused across cycles. pub fn new(pool: Arc, deps: HarnessDeps, record: CompanyRecord) -> Self { let responder = orchestrator::orchestrator_id(&record.manifest.agents).unwrap_or_default(); + let default_harness = record.manifest.default_harness_id(); + let bindings = record + .manifest + .agents + .iter() + .filter_map(|a| a.harness.clone().map(|h| (a.id.clone(), h))) + .collect(); Self { pool, - deps, + deps: Arc::new(deps), + lanes: Vec::new(), + bindings, + unavailable: Vec::new(), + default_harness, record: std::sync::RwLock::new(Arc::new(record)), responder, runs: None, @@ -300,6 +323,46 @@ impl HarnessBrain { } } + /// Attaches the harness lanes beyond the default. + /// + /// Each entry is a declared harness id and the engine serving it — another + /// `built_in` pool on its own provider, or an ACP agent. Without this a + /// brain routes every turn to the default lane, which is correct for a + /// company that declares no `[[harness]]`. + pub fn with_lanes(mut self, lanes: Vec<(String, Arc)>) -> Self { + self.lanes = lanes; + self + } + + /// Records that a declared harness has no engine on this host, and why, so + /// a turn bound to it fails with something actionable instead of silently + /// running somewhere nobody chose. + pub fn with_unavailable_lanes(mut self, unavailable: Vec<(String, String)>) -> Self { + self.unavailable = unavailable; + self + } + + /// The [`RunTurn`] this brain's turns go through. + /// + /// With no extra lanes this is the default harness alone — byte-identical + /// behaviour to before named harnesses existed, and no routing table is + /// consulted. With lanes, it is a [`HarnessRouter`] that sends each agent's + /// turn to the harness it is bound to. + fn run_turn(&self) -> Arc { + let default_lane: Arc = + Arc::new(HarnessRunTurn::new(self.pool.clone(), self.deps.clone())); + if self.lanes.is_empty() && self.unavailable.is_empty() { + return default_lane; + } + Arc::new(crate::harness::router::HarnessRouter::from_lanes( + &self.default_harness, + default_lane, + &self.lanes, + &self.unavailable, + &self.bindings, + )) + } + /// This company's record as of the current cycle's refresh. /// /// Returns a handle rather than a borrow so no lock is held across the @@ -487,7 +550,7 @@ impl HarnessBrain { ); let control = guard.control().clone(); - let run_turn = HarnessRunTurn::new(&self.pool, &self.deps); + let run_turn = self.run_turn(); // Bound for the runner's whole lifetime (issue #707): one turn, one record. let record = self.record(); // Issue #453: the same argument as the publish claim below, one queue @@ -601,7 +664,7 @@ impl HarnessBrain { // on this path. That is a strict improvement on dropping it unrun, and it // is recorded on the card the hand-off opens. let drained = match self - .delegation_runner(&run_turn, &record) + .delegation_runner(run_turn.as_ref(), &record) .drain_and_execute( grant.origin_thread.as_deref(), delegation::MessageContext::default(), @@ -843,7 +906,7 @@ impl HarnessBrain { let mut redirects: u32 = 0; // Route the background turn through the brain-agnostic `RunTurn` seam // (issue #176), re-attaching `HarnessDeps` behind `HarnessRunTurn`. - let run_turn = HarnessRunTurn::new(&self.pool, &self.deps); + let run_turn = self.run_turn(); // Bound for the runner's whole lifetime (issue #707): one turn, one record. let record = self.record(); // Issue #242: where this attempt's own approval requests begin. The @@ -934,7 +997,7 @@ impl HarnessBrain { // way to `todo` — the hand-off did happen, and a // re-dispatch should start from who it was given to. let handoff = match self - .delegation_runner(&run_turn, &record) + .delegation_runner(run_turn.as_ref(), &record) .for_task(&card.id) // The delegate's turn is part of THIS attempt — // its steps and its spend belong to the card's @@ -1127,7 +1190,7 @@ impl HarnessBrain { if !unpublished_before_nudge.is_empty() { declined = self .nudge_for_unpublished( - &run_turn, + run_turn.as_ref(), &responder, &base_instruction, &result_text, @@ -1566,7 +1629,7 @@ impl HarnessBrain { #[allow(clippy::too_many_arguments)] async fn nudge_for_unpublished( &self, - run_turn: &HarnessRunTurn<'_>, + run_turn: &dyn RunTurn, responder: &str, brief: &str, reply: &str, @@ -2588,9 +2651,9 @@ impl HarnessBrain { delegation: Delegation, chat_id: Option<&str>, ) -> Result { - let run_turn = HarnessRunTurn::new(&self.pool, &self.deps); + let run_turn = self.run_turn(); let record = self.record(); - self.delegation_runner(&run_turn, &record) + self.delegation_runner(run_turn.as_ref(), &record) .run_delegation(delegation, chat_id, delegation::MessageContext::default()) .await } @@ -2614,7 +2677,7 @@ impl HarnessBrain { /// turn on a single consistent record. fn delegation_runner<'a>( &'a self, - run_turn: &'a HarnessRunTurn<'a>, + run_turn: &'a dyn RunTurn, record: &'a CompanyRecord, ) -> DelegationRunner<'a> { DelegationRunner::new( @@ -2719,7 +2782,11 @@ impl HarnessBrain { host: &dyn CycleHost, ) -> Result { // Idempotent — builds the roster on the first cycle, a no-op after. - self.pool.ensure(&self.record(), &self.deps).await?; + // Warmed through the router, not the pool alone: a company with named + // harnesses has one pool per `built_in` harness, and each named lane's + // own pool must be populated before its first turn, or a bound agent + // fails with "company not found" while the default lane looks fine. + self.run_turn().ensure(&self.record()).await?; let mut channel_responses = Vec::new(); for event in &req.events { @@ -2825,11 +2892,11 @@ impl HarnessBrain { // orchestrator turn, its queued delegations, and the CEO-relay // hand-back all run behind the `RunTurn` impl. `HarnessDeps` is // re-attached behind `HarnessRunTurn`. - let run_turn = HarnessRunTurn::new(&self.pool, &self.deps); + let run_turn = self.run_turn(); // Bound for the runner's whole lifetime (issue #707): one turn, one record. let record = self.record(); let turn = self - .delegation_runner(&run_turn, &record) + .delegation_runner(run_turn.as_ref(), &record) // Issues #1035 / #1152: the operator's own statement of // what this message is for. The REST handler already // acts on it; until #1035 the runtime never saw it, so @@ -2900,7 +2967,7 @@ impl HarnessBrain { let nudge_control = SteerControl::new(); let declined = self .nudge_for_unpublished( - &run_turn, + run_turn.as_ref(), &responder, text, &operator_reply, @@ -3231,11 +3298,18 @@ description = "Runs Acme." } fn brain_over_mock(dir: &std::path::Path) -> HarnessBrain { + brain_over_mock_with(dir, record()) + } + + /// [`brain_over_mock`] over a chosen record, so a test can vary the roster + /// (and its `[[harness]]` block) without restating the whole deps literal. + fn brain_over_mock_with(dir: &std::path::Path, record: CompanyRecord) -> HarnessBrain { let deps = HarnessDeps { ledgers: None, ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(Arc::new(FsOps::new(dir))), @@ -3282,7 +3356,7 @@ description = "Runs Acme." repo_bindings: Vec::new(), checkouts: crate::harness::repo::CheckoutLedger::default(), }; - HarnessBrain::new(Arc::new(HarnessPool::new()), deps, record()) + HarnessBrain::new(Arc::new(HarnessPool::new()), deps, record) } fn request(events: Vec) -> CycleRequest { @@ -3419,6 +3493,7 @@ description = "Builds it." ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(Arc::new(FsOps::new(dir))), @@ -3543,6 +3618,7 @@ members = ["engineer"] ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(Arc::new(FsOps::new(dir))), @@ -4638,7 +4714,11 @@ members = ["engineer"] let dir = tempfile::tempdir().unwrap(); let (mut brain, _ops) = brain_with_artifacts(dir.path()); - brain.deps.artifacts = Some(Arc::new(BrokenArtifacts)); + // Sole owner at this point — no turn has run, so nothing has cloned the + // deps into a lane yet. + Arc::get_mut(&mut brain.deps) + .expect("the brain is the only holder of its deps before any turn") + .artifacts = Some(Arc::new(BrokenArtifacts)); let err = brain .record_published_artifacts( @@ -5443,6 +5523,7 @@ members = ["engineer"] ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(Arc::new(FsOps::new(dir))), @@ -6433,6 +6514,7 @@ members = ["eng1", "eng2"] ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir.path())), store: Arc::new(FsCompanyStore::new(dir.path())), meter: None, @@ -6584,6 +6666,7 @@ members = ["eng1", "eng2"] ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir.path())), store: Arc::new(FsCompanyStore::new(dir.path())), meter: None, @@ -6678,6 +6761,7 @@ members = ["eng1", "eng2"] ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, @@ -7013,6 +7097,7 @@ members = ["eng1", "eng2"] ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, @@ -7516,6 +7601,7 @@ members = ["eng1", "eng2"] extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, @@ -7841,6 +7927,7 @@ members = ["eng1", "eng2"] ledger_registry: Default::default(), provider: provider.clone(), provider_slug: "steering".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, @@ -8247,6 +8334,7 @@ members = ["eng1", "eng2"] ledger_registry: Default::default(), provider: provider.clone(), provider_slug: "delegating".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, @@ -8863,6 +8951,195 @@ members = ["eng1", "eng2"] assert_eq!(parent.column, "in_review"); } + // ---- named harnesses: does the wiring actually route? ------------------ + + /// Records which agents it ran, so a test can assert *which* lane served a + /// turn rather than only that one did. + struct SpyLane { + label: String, + seen: std::sync::Mutex>, + } + + #[async_trait] + impl crate::runtime::delegation::RunTurn for SpyLane { + async fn run( + &self, + _company: &CompanyId, + agent_id: &str, + _message: &str, + _chat_id: Option<&str>, + ) -> Result { + self.seen.lock().unwrap().push(agent_id.to_string()); + Ok(crate::harness::built_in::TurnOutcome { + reply: self.label.clone(), + steps: Vec::new(), + hit_iteration_cap: false, + halted_for_spend: None, + }) + } + async fn run_steered( + &self, + c: &CompanyId, + a: &str, + m: &str, + _: &crate::company::steer::SteerControl, + chat: Option<&str>, + _: Option>, + ) -> Result { + self.run(c, a, m, chat).await + } + async fn run_steered_background( + &self, + c: &CompanyId, + a: &str, + m: &str, + _: &crate::company::steer::SteerControl, + _: Option>, + ) -> Result { + self.run(c, a, m, None).await + } + } + + /// A roster spanning two declared harnesses. + fn two_harness_record() -> CompanyRecord { + let manifest = toml::from_str( + r#" +[company] +name = "Acme" + +[policy] +mode = "full" + +[[agent]] +id = "ceo" +role = "Chief Executive" + +[[agent]] +id = "researcher" +role = "Researcher" +harness = "deep" + +[[harness]] +id = "embedded" +kind = "built_in" +default = true + +[[harness]] +id = "deep" +kind = "built_in" +"#, + ) + .expect("valid manifest"); + CompanyRecord { + manifest, + ..record() + } + } + + /// The wiring's whole point: an agent bound to a named harness runs on that + /// harness's engine, and an unbound one stays on the default. + #[tokio::test] + async fn a_bound_agent_runs_on_its_own_lane() { + let dir = tempfile::tempdir().unwrap(); + let deep = Arc::new(SpyLane { + label: "deep".to_string(), + seen: std::sync::Mutex::new(Vec::new()), + }); + let brain = brain_over_mock_with(dir.path(), two_harness_record()).with_lanes(vec![( + "deep".to_string(), + deep.clone() as Arc, + )]); + + let company = CompanyId::new("acme"); + let out = brain + .run_turn() + .run(&company, "researcher", "hi", None) + .await + .expect("routes to the deep lane"); + assert_eq!(out.reply, "deep"); + assert_eq!(&*deep.seen.lock().unwrap(), &["researcher".to_string()]); + + // The unbound agent must not reach it — it belongs to the default pool. + let _ = brain.run_turn().run(&company, "ceo", "hi", None).await; + assert_eq!( + &*deep.seen.lock().unwrap(), + &["researcher".to_string()], + "the default agent must not land on the named lane" + ); + } + + /// The named lane is a **real** pool, not a spy: it must build its own + /// roster at boot, or a bound agent's first turn fails `CompanyNotFound` + /// on an empty pool. This is the path the `SpyLane` coverage above cannot + /// reach — a spy forwards every turn, so it would pass whether or not the + /// lane's pool was ever warmed. + #[tokio::test] + async fn a_named_lane_builds_its_roster_at_boot() { + let dir = tempfile::tempdir().unwrap(); + let brain = brain_over_mock_with(dir.path(), two_harness_record()); + // The lane `lanes::build` produces: its own pool over deps narrowed to + // the agents it serves. + let mut deep_deps = (*brain.deps).clone(); + deep_deps.serves = Some(std::collections::HashSet::from(["researcher".to_string()])); + let deep: Arc = Arc::new(HarnessRunTurn::new( + Arc::new(HarnessPool::new()), + Arc::new(deep_deps), + )); + let brain = brain.with_lanes(vec![("deep".to_string(), deep.clone())]); + + // Boot warm-up: the router warms every lane's engine, each against its + // own narrowed deps. + brain + .run_turn() + .ensure(&brain.record()) + .await + .expect("every lane's roster builds"); + + // A bound agent's turn now reaches its lane's engine instead of dying + // with "company not found". + let out = brain + .run_turn() + .run(&CompanyId::new("acme"), "researcher", "hi", None) + .await + .expect("the deep lane's roster is built at boot"); + assert!(out.reply.contains("hi"), "{}", out.reply); + } + + /// A declared harness this host cannot run fails the turn, naming the + /// harness and the reason. It must never quietly borrow the default lane: + /// that turn would succeed on a model and a credential nobody chose, and + /// the only evidence would be a billing line. + #[tokio::test] + async fn an_unrunnable_harness_fails_rather_than_falling_back() { + let dir = tempfile::tempdir().unwrap(); + let brain = + brain_over_mock_with(dir.path(), two_harness_record()).with_unavailable_lanes(vec![( + "deep".to_string(), + "this build has no ACP transport wired".to_string(), + )]); + + let err = brain + .run_turn() + .run(&CompanyId::new("acme"), "researcher", "hi", None) + .await + .expect_err("must not fall back to the default lane"); + let msg = err.to_string(); + assert!(msg.contains("researcher"), "{msg}"); + assert!(msg.contains("deep"), "{msg}"); + assert!(msg.contains("ACP transport"), "names the fix: {msg}"); + } + + /// A company declaring no `[[harness]]` keeps exactly the single-lane path: + /// no lanes, no bindings, nothing to consult. + #[tokio::test] + async fn a_company_with_no_harness_block_is_unrouted() { + let dir = tempfile::tempdir().unwrap(); + let brain = brain_over_mock(dir.path()); + assert!(brain.lanes.is_empty()); + assert!(brain.unavailable.is_empty()); + assert!(brain.bindings.is_empty()); + assert_eq!(brain.default_harness, "default"); + } /// Issue #966: a workflow-copilot reply is authored by the copilot. /// /// This is the assertion the #885 fix was missing on this branch. The bubble diff --git a/src/harness/build.rs b/src/harness/built_in/build.rs similarity index 99% rename from src/harness/build.rs rename to src/harness/built_in/build.rs index 13ee9376c..bc3117f06 100644 --- a/src/harness/build.rs +++ b/src/harness/built_in/build.rs @@ -1058,10 +1058,13 @@ pub fn build_agent( // always kept. let tools = toolbelt::filter_by_capabilities(tools, &deps.capabilities); let tools = if deps.workspace_git_enabled { - match crate::harness::checkpoint::WorkspaceCheckpointer::initialize_off_worker(&workspace) { - Ok(checkpointer) => { - crate::harness::checkpoint::CheckpointingTool::wrap_all(tools, checkpointer) - } + match crate::harness::built_in::checkpoint::WorkspaceCheckpointer::initialize_off_worker( + &workspace, + ) { + Ok(checkpointer) => crate::harness::built_in::checkpoint::CheckpointingTool::wrap_all( + tools, + checkpointer, + ), Err(error) => { tracing::warn!( company = %company, @@ -1586,6 +1589,7 @@ mod tests { name: None, description: description.map(str::to_string), tier: None, + harness: None, tools: Vec::new(), delegates_to: Vec::new(), context: None, @@ -1707,6 +1711,7 @@ mod tests { ledger_registry: Default::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(PinContext), store: Arc::new(PinStore), meter: None, @@ -1784,6 +1789,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: Vec::new(), delegates_to: delegates_to.iter().map(|d| d.to_string()).collect(), context: None, @@ -1833,6 +1839,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: Vec::new(), delegates_to: Vec::new(), context: None, @@ -1878,6 +1885,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: Vec::new(), delegates_to: Vec::new(), context: None, @@ -1921,6 +1929,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: Vec::new(), delegates_to: Vec::new(), context: None, @@ -2123,6 +2132,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: Vec::new(), // Issue #176: this repo-tools fixture predates `delegates_to` and is // only compiled under the gated feature combo, so neither this @@ -2573,6 +2583,7 @@ mod tests { tools: Vec::new(), delegates_to: Vec::new(), context: None, + harness: None, budget_usd_daily: None, prompt: None, prompt_files: Vec::new(), @@ -2827,6 +2838,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: Vec::new(), delegates_to: Vec::new(), context: None, diff --git a/src/harness/capability_budget.rs b/src/harness/built_in/capability_budget.rs similarity index 100% rename from src/harness/capability_budget.rs rename to src/harness/built_in/capability_budget.rs diff --git a/src/harness/chargebee.rs b/src/harness/built_in/chargebee.rs similarity index 100% rename from src/harness/chargebee.rs rename to src/harness/built_in/chargebee.rs diff --git a/src/harness/checkpoint.rs b/src/harness/built_in/checkpoint.rs similarity index 100% rename from src/harness/checkpoint.rs rename to src/harness/built_in/checkpoint.rs diff --git a/src/harness/composio.rs b/src/harness/built_in/composio.rs similarity index 100% rename from src/harness/composio.rs rename to src/harness/built_in/composio.rs diff --git a/src/harness/composio_catalog.rs b/src/harness/built_in/composio_catalog.rs similarity index 100% rename from src/harness/composio_catalog.rs rename to src/harness/built_in/composio_catalog.rs diff --git a/src/harness/composio_turn_test.rs b/src/harness/built_in/composio_turn_test.rs similarity index 99% rename from src/harness/composio_turn_test.rs rename to src/harness/built_in/composio_turn_test.rs index 82566fd4e..eac963886 100644 --- a/src/harness/composio_turn_test.rs +++ b/src/harness/built_in/composio_turn_test.rs @@ -329,6 +329,7 @@ async fn harness( extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, diff --git a/src/harness/confine.rs b/src/harness/built_in/confine.rs similarity index 100% rename from src/harness/confine.rs rename to src/harness/built_in/confine.rs diff --git a/src/harness/cost.rs b/src/harness/built_in/cost.rs similarity index 95% rename from src/harness/cost.rs rename to src/harness/built_in/cost.rs index 5999fc03e..e9e18deb0 100644 --- a/src/harness/cost.rs +++ b/src/harness/built_in/cost.rs @@ -118,7 +118,18 @@ pub async fn record_turn_cost( } if let (Some(meter), Some(sample)) = (meter, usage_sample_for(turn, agent_id, provider, run_id)) { - meter.record(company, &sample).await?; + // The usage sample is telemetry, not the turn's record of itself: the + // ledger write above has already happened, so a meter failure must not + // fail a completed turn. Log it and let the turn stand. + if let Err(error) = meter.record(company, &sample).await { + tracing::warn!( + company = %company, + agent = %agent_id, + provider = %provider, + error = %error, + "[cost] failed to record the usage sample; the turn still stands" + ); + } } Ok(()) } diff --git a/src/harness/embeddings.rs b/src/harness/built_in/embeddings.rs similarity index 100% rename from src/harness/embeddings.rs rename to src/harness/built_in/embeddings.rs diff --git a/src/harness/hosting.rs b/src/harness/built_in/hosting.rs similarity index 100% rename from src/harness/hosting.rs rename to src/harness/built_in/hosting.rs diff --git a/src/harness/iteration_cap_turn_test.rs b/src/harness/built_in/iteration_cap_turn_test.rs similarity index 99% rename from src/harness/iteration_cap_turn_test.rs rename to src/harness/built_in/iteration_cap_turn_test.rs index 07843b346..e54948214 100644 --- a/src/harness/iteration_cap_turn_test.rs +++ b/src/harness/built_in/iteration_cap_turn_test.rs @@ -195,6 +195,7 @@ fn deps(model_url: String, dir: &std::path::Path) -> HarnessDeps { extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, @@ -272,6 +273,7 @@ async fn company_agent( tools: Vec::new(), delegates_to: Vec::new(), context: None, + harness: None, budget_usd_daily, prompt: None, prompt_files: Vec::new(), diff --git a/src/harness/ledger_tools.rs b/src/harness/built_in/ledger_tools.rs similarity index 96% rename from src/harness/ledger_tools.rs rename to src/harness/built_in/ledger_tools.rs index 71de456bc..3db1c62b6 100644 --- a/src/harness/ledger_tools.rs +++ b/src/harness/built_in/ledger_tools.rs @@ -155,12 +155,24 @@ pub fn ledger_brief(registry: &crate::ledger::Registry) -> String { brief } -/// Resolves a slug, or an error naming the ones that exist. -async fn spec_for(ctx: &Ledgers, arguments: &Value) -> Result { +/// Resolves a slug the agent may use, or an error naming the ones it may not. +/// +/// Access is checked on the **raw** slug before the registry is opened, so an +/// agent without a grant never learns that a ledger exists: the refusal is the +/// same whether the ledger is real or not. The registry's "here are all the +/// slugs" error is only reachable by an agent whose grant already names the +/// ledger. +async fn spec_for( + ctx: &Ledgers, + arguments: &Value, + grants: &Option>, + need: LedgerAccess, +) -> Result { let slug = text(arguments, "ledger"); if slug.is_empty() { return Err("Name the `ledger` to use. `list_ledgers` names them all.".to_string()); } + require_access(grants, &slug, need)?; let registry = ledgers::registry(ctx) .await .map_err(|error| format!("Could not read this company's ledgers: {error}."))?; @@ -374,13 +386,17 @@ impl Tool for ReadLedger { } async fn execute(&self, arguments: Value) -> anyhow::Result { - let spec = match spec_for(&self.ctx, &arguments).await { + let spec = match spec_for( + &self.ctx, + &arguments, + &self.ledger_grants, + LedgerAccess::Read, + ) + .await + { Ok(spec) => spec, Err(message) => return Ok(ToolResult::error(message)), }; - if let Err(message) = require_access(&self.ledger_grants, &spec.slug, LedgerAccess::Read) { - return Ok(ToolResult::error(message)); - } let query = Query { entry: optional(&arguments, "entry"), status: optional(&arguments, "status"), @@ -482,14 +498,17 @@ impl Tool for RecordEntry { } async fn execute(&self, arguments: Value) -> anyhow::Result { - let spec = match spec_for(&self.ctx, &arguments).await { + let spec = match spec_for( + &self.ctx, + &arguments, + &self.ledger_grants, + LedgerAccess::Record, + ) + .await + { Ok(spec) => spec, Err(message) => return Ok(ToolResult::error(message)), }; - if let Err(message) = require_access(&self.ledger_grants, &spec.slug, LedgerAccess::Record) - { - return Ok(ToolResult::error(message)); - } let id = text(&arguments, "id"); match ledgers::record( &self.ctx, @@ -566,14 +585,17 @@ impl Tool for CloseEntry { } async fn execute(&self, arguments: Value) -> anyhow::Result { - let spec = match spec_for(&self.ctx, &arguments).await { + let spec = match spec_for( + &self.ctx, + &arguments, + &self.ledger_grants, + LedgerAccess::Record, + ) + .await + { Ok(spec) => spec, Err(message) => return Ok(ToolResult::error(message)), }; - if let Err(message) = require_access(&self.ledger_grants, &spec.slug, LedgerAccess::Record) - { - return Ok(ToolResult::error(message)); - } match ledgers::close( &self.ctx, &spec, diff --git a/src/harness/ledger_tools_test.rs b/src/harness/built_in/ledger_tools_test.rs similarity index 100% rename from src/harness/ledger_tools_test.rs rename to src/harness/built_in/ledger_tools_test.rs diff --git a/src/harness/lifecycle.rs b/src/harness/built_in/lifecycle.rs similarity index 100% rename from src/harness/lifecycle.rs rename to src/harness/built_in/lifecycle.rs diff --git a/src/harness/mcp.rs b/src/harness/built_in/mcp.rs similarity index 99% rename from src/harness/mcp.rs rename to src/harness/built_in/mcp.rs index 1f0b76e85..de3913c8b 100644 --- a/src/harness/mcp.rs +++ b/src/harness/built_in/mcp.rs @@ -661,6 +661,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: grants.iter().map(|g| g.to_string()).collect(), delegates_to: vec![], context: None, diff --git a/src/harness/mcp_probe.rs b/src/harness/built_in/mcp_probe.rs similarity index 100% rename from src/harness/mcp_probe.rs rename to src/harness/built_in/mcp_probe.rs diff --git a/src/harness/memory.rs b/src/harness/built_in/memory.rs similarity index 80% rename from src/harness/memory.rs rename to src/harness/built_in/memory.rs index 4d35c368f..136c396e2 100644 --- a/src/harness/memory.rs +++ b/src/harness/built_in/memory.rs @@ -30,6 +30,7 @@ //! A `tinycortex`-backed [`ContextStore`] removes these gaps; the adapter code //! is identical because it only speaks the port. +use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; @@ -38,7 +39,7 @@ use openhuman_core::openhuman as oh; use oh::memory::{Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts}; use crate::ports::ContextStore; -use crate::ports::types::{CompanyId, ContextChunk}; +use crate::ports::types::{ChunkAddr, CompanyId, ContextChunk}; /// openhuman [`Memory`] backed by an opencompany [`ContextStore`], namespaced to /// one `{company}/{agent}` pair. @@ -141,30 +142,59 @@ impl Memory for OcMemory { limit: usize, opts: RecallOpts<'_>, ) -> anyhow::Result> { + // `search` scans the whole company, so a hit is only this agent's when + // its addr carries one of our labels. Fetch the in-scope label set once + // and filter by addr; the stored `{ns}/{key}` tail also corrects the + // entry metadata, which previously reported the requested namespace + // with an empty key. + let prefix = match opts.namespace { + Some(ns) => self.namespace_prefix(ns), + None => format!("{}/", self.scope()), + }; + let in_scope: HashMap = self + .context + .list(&self.company, &prefix) + .await + .map_err(|e| anyhow::anyhow!("context list failed: {e}"))? + .into_iter() + .map(|meta| (meta.addr, meta.label)) + .collect(); let hits = self .context .search(&self.company, query, limit) .await .map_err(|e| anyhow::anyhow!("context search failed: {e}"))?; let min = opts.min_score.unwrap_or(0.0); - let namespace = opts.namespace.unwrap_or("global"); Ok(hits .into_iter() .filter(|h| h.score >= min) - .map(|h| { + .filter_map(|h| { + let label = in_scope.get(&h.addr)?; + let (ns, key) = self.split_label(label)?; let id: String = h.addr.as_ref().to_string(); - self.entry_from(id, namespace, "", h.snippet, Some(h.score)) + Some(self.entry_from(id, ns, key, h.snippet, Some(h.score))) }) .collect()) } async fn recall_relevant_by_vector( &self, - _namespace: &str, + namespace: &str, query: &str, limit: usize, min_vector_similarity: f64, ) -> anyhow::Result> { + // Same scoping rule as `recall`: `search` scans the whole company, so a + // hit is only relevant when its addr is one of ours. The label set is + // read once, before the search, so a degrading search still filters. + let in_scope: std::collections::HashSet = self + .context + .list(&self.company, &self.namespace_prefix(namespace)) + .await + .map_err(|e| anyhow::anyhow!("context list failed: {e}"))? + .into_iter() + .map(|meta| meta.addr) + .collect(); // fs backend has no vectors — degrade to substring/FTS search and never // error (per the trait contract). A failed search yields an empty recall. let hits = match self.context.search(&self.company, query, limit).await { @@ -177,6 +207,7 @@ impl Memory for OcMemory { Ok(hits .into_iter() .filter(|h| h.score >= min_vector_similarity) + .filter(|h| in_scope.contains(&h.addr)) .map(|h| { let addr: String = h.addr.as_ref().to_string(); (addr, h.snippet) @@ -384,12 +415,55 @@ mod tests { ceo.store("global", "secret", "ceo-only", MemoryCategory::Core, None) .await .unwrap(); + cfo.store("global", "secret", "cfo-only", MemoryCategory::Core, None) + .await + .unwrap(); // The CFO shares the company + ContextStore but must not see the CEO's - // namespaced entry. - assert!(cfo.get("global", "secret").await.unwrap().is_none()); + // namespaced entry. Each agent reads back its *own* value — the CEO's + // "ceo-only" content is isolated under the CEO's agent scope and never + // surfaces for the CFO. + let cfo_got = cfo + .get("global", "secret") + .await + .unwrap() + .expect("the CFO stored this key itself"); + assert_eq!(cfo_got.content, "cfo-only"); + let ceo_got = ceo + .get("global", "secret") + .await + .unwrap() + .expect("the CEO stored this key itself"); + assert_eq!(ceo_got.content, "ceo-only"); assert_eq!(ceo.count().await.unwrap(), 1); - assert_eq!(cfo.count().await.unwrap(), 0); + assert_eq!(cfo.count().await.unwrap(), 1); + + // Both recall methods are scoped the same way: a query both entries + // match surfaces only the caller's own chunk, and the recalled entry + // carries the stored namespace and key rather than a substituted + // default. + let ceo_hits = ceo + .recall( + "only", + 5, + RecallOpts { + namespace: Some("global"), + ..RecallOpts::default() + }, + ) + .await + .unwrap(); + assert_eq!(ceo_hits.len(), 1); + assert_eq!(ceo_hits[0].content, "ceo-only"); + assert_eq!(ceo_hits[0].namespace.as_deref(), Some("global")); + assert_eq!(ceo_hits[0].key, "secret"); + + let cfo_vec = cfo + .recall_relevant_by_vector("global", "only", 5, 0.0) + .await + .unwrap(); + assert_eq!(cfo_vec.len(), 1); + assert_eq!(cfo_vec[0].1, "cfo-only"); } #[tokio::test] @@ -410,6 +484,10 @@ mod tests { .unwrap(); assert_eq!(hits.len(), 1); assert!(hits[0].content.contains("quarterly")); + // The recalled entry answers with the stored label's tail, not the + // requested default: namespace "global" and the real key. + assert_eq!(hits[0].namespace.as_deref(), Some("global")); + assert_eq!(hits[0].key, "note"); } #[tokio::test] diff --git a/src/harness/memory_loop.rs b/src/harness/built_in/memory_loop.rs similarity index 88% rename from src/harness/memory_loop.rs rename to src/harness/built_in/memory_loop.rs index a6fd2356b..c3a32d952 100644 --- a/src/harness/memory_loop.rs +++ b/src/harness/built_in/memory_loop.rs @@ -60,7 +60,12 @@ fn inject_within(message: &str, hits: &[ChunkHit], history_budget: usize) -> Str let mut remaining = history_budget; let mut skipped = 0usize; for hit in hits { - let snippet = truncate_chars(hit.snippet.trim(), MAX_SNIPPET_CHARS); + // Flatten whitespace (split_whitespace / join(" ")) so embedded newlines + // in stored snippets cannot cross the `## Task` boundary below. + let snippet = truncate_chars( + &hit.snippet.split_whitespace().collect::>().join(" "), + MAX_SNIPPET_CHARS, + ); let cost = snippet.chars().count() + 3; // "- " + "\n" if cost > remaining { // Skip, don't stop: a later, smaller hit may still fit, and every @@ -251,4 +256,26 @@ mod test { assert!(chunk.body.contains("plan the launch")); assert!(chunk.body.contains("here is the plan")); } + + #[test] + fn multiline_snippets_are_flattened_before_injection() { + // Embedded newlines in a stored snippet must be collapsed before + // truncation; otherwise they cross the `## Task` boundary and break + // the preamble structure (CWE-74 — injection of untrusted whitespace + // into the prompt format). + let out = inject("now", &[hit("Task: plan\nOutcome: drafted\n\nNotes: good")]); + let preamble = out.split("## Task").next().unwrap(); + let injected_line = preamble + .lines() + .find(|l| l.starts_with("- ")) + .expect("the multiline snippet produces an injected line"); + assert!( + !injected_line.contains('\n'), + "newlines must be collapsed, got: {injected_line:?}" + ); + assert!( + injected_line.contains("Task: plan Outcome: drafted Notes: good"), + "all whitespace runs are collapsed to single spaces: {injected_line:?}" + ); + } } diff --git a/src/harness/built_in/mod.rs b/src/harness/built_in/mod.rs new file mode 100644 index 000000000..8ce4fa38c --- /dev/null +++ b/src/harness/built_in/mod.rs @@ -0,0 +1,7052 @@ +//! WS4 — openhuman embedded as a library (the harness). +//! +//! This module supersedes the out-of-process OpenHuman seam +//! (`src/openhuman/{launcher,rpc,tools,channel}.rs`, JSON-RPC behind +//! `openhuman-rpc`) with **direct library embedding** of `vendor/openhuman` +//! (`openhuman_core`): one openhuman [`Agent`](oh::agent::Agent) per manifest +//! `[[agent]]`, wired with memory, an inference provider, an approval policy, +//! and a workspace through [`AgentBuilder`](oh::agent::AgentBuilder). +//! +//! Compiled only under `feature = "openhuman"`. The default build links none of +//! it and keeps its offline, echo-brained behaviour. +//! +//! ## Layout +//! +//! * [`build`] — manifest `[[agent]]` → `AgentBuilder`. +//! * [`provider`] — hosted Medulla [`Provider`] + a `MockProvider` for tests. +//! * [`memory`] — [`OcMemory`](memory::OcMemory): openhuman `Memory` over the +//! opencompany [`ContextStore`](crate::ports::ContextStore). +//! * [`policy`] — [`ApprovalPolicy`](policy::ApprovalPolicy): `[policy]` → +//! openhuman `ToolPolicy`. +//! * [`cost`] — [`TurnCost`](oh::agent::cost::TurnCost) → ledger + usage meter. +//! +//! ## Flagged seams +//! +//! * **Group-chat / desk routing** is opencompany's job (openhuman is +//! single-agent). v1 is single-responder; the full ops `chat` handler that +//! resolves a desk's members and journals the `AgentReply` is WS3. +//! +//! Live turn cost is **wired**: [`CompanyAgent::run`] reads the completed turn's +//! token/cost totals from openhuman's public +//! [`Agent::last_turn_usage`](oh::agent::Agent::last_turn_usage) accessor and +//! [`HarnessPool::run`] records them through [`cost::record_turn_cost`]. Usage +//! only reaches the ledger/meter when the provider reports it — the +//! [`HostedProvider`](provider::HostedProvider) parses it off the wire; the +//! offline [`MockProvider`](provider::MockProvider) does not, so test turns stay +//! inert. + +/// Issue #775: the fail-closed shell audit wrapper — one intent line appended +/// (and fsynced) *before* a command runs, refusing the command outright when +/// that append fails. Pairs with the host-owned, per-agent sink +/// [`toolbelt::shell_audit`] resolves. See [`audit`]. +pub mod audit; +pub mod brain; +pub mod build; +pub mod capability_budget; +#[cfg(feature = "chargebee")] +pub mod chargebee; +mod checkpoint; +pub mod composio; +/// Issue #410: how a Composio action catalogue is narrowed and rendered for an +/// agent, and why every cut it makes describes itself. Pure and un-gated (the +/// live tools are behind `composio`, which CI never *runs*) — see +/// [`composio_catalog`]. +pub mod composio_catalog; +/// End-to-end proof that #410's narrowable, self-describing Composio listing is +/// reachable from a real turn on two large toolkits — the harness, the grant +/// gate, the approval policy and the Composio client are all real; only the +/// model's choices and the Composio backend are scripted. Test-only. +#[cfg(all(test, feature = "composio"))] +mod composio_turn_test; +/// Issue #416: the confined turn — an ephemeral agent with no tools, no company +/// memory and no delegation, for a question that is about one object rather than +/// about the company. See [`confine`]. +pub mod confine; +pub mod cost; +/// Hosted embeddings compute for the in-pod memory engine's meaning tier (188c2). +/// Needs the `tinycortex` crate's `EmbeddingBackend` trait, so it links only when +/// both the harness (`openhuman`) and the memory engine (`tinycortex`) are built. +#[cfg(feature = "tinycortex")] +pub mod embeddings; +/// Hosting (TinyHosts): the per-company connection and the agent tools over it. +/// The keys it reads live in `company::hosting`, which is compiled in every +/// build — the console's Hosting settings write them whether or not this +/// harness exists to use them. +pub mod hosting; +/// End-to-end proof of issue #988: a turn really does get +/// [`MAX_TOOL_ITERATIONS`](build::MAX_TOOL_ITERATIONS) tool rounds instead of the +/// vendored ten, and a budget-armed turn's in-turn +/// [`BudgetStopHook`](oh::agent::stop_hooks::BudgetStopHook) halts it when it +/// outruns its money — distinguishably from an iteration-cap pause. Test-only. +/// +/// Declared here rather than at `crate::harness` because it reads +/// `CompanyAgent`'s private `agent` field (the vendored session) to ask +/// `last_turn_hit_cap` — a child of `built_in`, not of the re-exporting parent. +#[cfg(test)] +mod iteration_cap_turn_test; +pub mod ledger_tools; +pub mod lifecycle; +pub mod mcp; +pub mod mcp_probe; +pub mod memory; +pub mod memory_loop; +pub mod orchestrator; +/// Chargebee billing tools (issue #788), wired per company from its own +/// SecretStore. Always compiled so the credential resolution and the fail-closed +/// decision are testable at default features; only the tools are gated. +/// PayPal wallet + transaction tools (issue #789), wired per company from its +/// own SecretStore. Always compiled so credential resolution and the +/// fail-closed decision are testable at default features. +#[cfg(feature = "paypal")] +pub mod paypal; +/// Issue #337: the planning station — one tool-less model call per card entering +/// `planning`, with the host gathering the evidence and verifying every +/// prerequisite the model claims. See [`planning`]. +pub mod planning; +pub mod policy; +pub mod provider; +/// Issue #244: `publish_artifact` — the only way a workspace file becomes a +/// deliverable — plus the staging queue the brain drains, the bounded workspace +/// scan that detects unpublished work, and the follow-up nudge's prompt. See +/// [`publish`]. +pub mod publish; +/// End-to-end proof that #244's `publish_artifact` is reachable from a real +/// dispatch, that a re-run extends by identity, and — the part nothing shorter +/// than a real turn loop can show — that the follow-up nudge fires **once**, +/// records a decline, and can never fail the run it follows. Test-only. +#[cfg(test)] +mod publish_turn_test; +/// Issue #245, agent half: `repo_checkout` / `repo_pr` behind an explicit +/// `repo` grant — a **confined** working tree cloned out of the host's mirror +/// (a full object copy, then every reference back to the mirror severed), plus +/// the per-turn ledger that deletes it again. See [`repo`]. +pub mod repo; +pub mod run_trace; +pub mod run_turn; +pub mod search; +/// End-to-end proof that the #238 `web_search` tool is reachable from a real +/// turn — the harness, the grant gates, the approval policy, the cap and the +/// meter are all real; only the model's choices and the search backend's +/// responses are scripted. Test-only. +#[cfg(test)] +mod search_turn_test; +pub mod skills; +pub mod steer; +pub mod steps; +pub mod tool_dispatcher; +pub mod toolbelt; +pub mod triage; +/// Issue #661 (M7): `read_workflow` / `update_workflow` / `delete_workflow` — +/// the agent's way to fix or retire a workflow instead of only ever creating +/// another one beside it. Kept out of `orchestrator.rs` (already the largest +/// file in `src/harness/`) because the three share a handle, a guard and a set +/// of refusals with each other rather than with anything there. See +/// [`workflow_admin`]. +pub mod workflow_admin; +/// Issue #339: the staging queue the orchestrator's `run_workflow` / +/// `create_workflow` tools push a workflow reference onto and the +/// [`HarnessBrain`] drains at the end of a dispatch, so a card that built or +/// Issue #580: the workflow builder pass — turns a `workflow`-deliverable card's +/// plan into a proposed graph that lands In Review for approval. Modeled on the +/// planning station (one card, one tool-less model call, one settled outcome), +/// but it mints an attempt row because building the workflow is the card's work. +/// See [`workflow_build`]. +pub mod workflow_build; +/// ran a workflow can link to it. See [`workflow_refs`]. +pub mod workflow_refs; +/// End-to-end proof that an agent granted `files` and **not** `shell` can write +/// a relative path on a company that has never run — the #409 provisioning gap, +/// which only exists before anything has created the agent's workspace. Covers +/// a manifest teammate and a runtime overlay teammate, and pins that a traversal +/// out of a provisioned sandbox is still refused. Test-only. +#[cfg(test)] +mod workspace_provision_turn_test; +pub mod workspace_tools; +/// End-to-end proof that the #237 workspace tools are reachable from a real +/// turn, with only the model's choices stubbed. Test-only. +#[cfg(test)] +mod workspace_turn_test; + +pub use brain::HarnessBrain; + +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::Arc; + +use openhuman_core::openhuman as oh; +use tokio::sync::{Mutex, RwLock}; + +use oh::agent::Agent; + +use crate::harness::provider::HarnessModel; + +use crate::company::Agent as ManifestAgent; +use crate::company::Policy; +use crate::company::mcp::McpServerDecl; +use crate::company::steer::{SteerAction, SteerControl}; +use crate::error::OpenCompanyError; +use crate::harness::cost::{TurnUsage, record_turn_cost}; +use crate::harness::mcp_probe::McpFailureQueue; +use crate::harness::orchestrator::DelegationQueue; +use crate::harness::policy::{ApprovalPolicy, ApprovalRequestQueue}; +use crate::ports::skills_state::{SkillState, SkillStateStore}; +use crate::ports::types::{ + BudgetOverride, CompanyId, CompanyRecord, OverlayAgent, OverlayDesk, OverlayDeskMember, + PolicyOverride, TurnStep, +}; +use crate::ports::{ + ArtifactStore, CompanyStore, ContextStore, EventLog, FactStore, SecretStore, TaskStore, + UsageMeter, +}; +use crate::runtime::builder::agent_scoped_grants; + +/// Shared dependencies every harness-built agent draws on. +#[derive(Clone)] +pub struct HarnessDeps { + /// The inference model shared across a company's agents. A [`HarnessModel`] + /// is a tinyagents [`ChatModel<()>`](tinyagents::harness::model::ChatModel) + /// plus the telemetry slug the cost hook reads live per turn; it upcasts to + /// `Arc>` at the openhuman `AgentBuilder::chat_model` seam. + pub provider: Arc, + /// Stable provider slug attributed to usage samples (e.g. `subscription`). + pub provider_slug: String, + /// Which agents this pool builds, when it serves one named harness rather + /// than the whole company. + /// + /// `None` — the whole roster, which is every pre-harness caller and the + /// single-harness case. + /// + /// `Some(ids)` — only those agents. A company running two `built_in` + /// harnesses gets one pool per harness, each holding its own + /// [`provider`](Self::provider); without this filter every pool would build + /// every agent, so a ten-agent roster on three harnesses would stand up + /// thirty live agents to use ten. + pub serves: Option>, + /// Context store backing every agent's [`OcMemory`](memory::OcMemory). + pub context: Arc, + /// Company store the cost hook appends ledger entries to. + pub store: Arc, + /// Optional usage meter (WS5 seam); `None` skips usage sampling. + pub meter: Option>, + /// Root under which per-agent workspace directories are created + /// (`{root}/{company}/{agent}/workspace`). + pub workspace_root: PathBuf, + /// Whether each private agent workspace is initialized as a Git repository + /// and checkpointed after tool calls. Host-level `[workspace]` config owns + /// this switch; false preserves the pre-checkpoint behavior exactly. + pub workspace_git_enabled: bool, + /// The **instance data root** the shell audit sink hangs off, resolved + /// through [`DataLayout::agent_audit_dir`](crate::store::DataLayout::agent_audit_dir) + /// to `companies//audit//` (issue #775). + /// + /// Carried as its own field rather than derived from + /// [`workspace_root`](Self::workspace_root)`.parent()` on purpose. The two + /// are siblings under one data root today, and an implicit + /// `workspace_root/..` would make a security boundary depend on a directory + /// relationship nobody declared — the ambient-context coupling this codebase + /// keeps getting bitten by. The audit sink is where it is because a caller + /// said so. + /// + /// It must never be inside `workspace_root`: the agent workspace is also the + /// `workspace_only` `SecurityPolicy` root the file tools enforce, so a sink + /// under it is a policy-*permitted* write target for the very agent it + /// records. + pub audit_root: PathBuf, + /// Optional model/tier applied to every agent, overriding the per-agent + /// `tier` → model mapping. Set from the resolved hosted-inference model so + /// the whole roster addresses the configured workload (e.g. `chat-v1`). + /// `None` keeps each agent's tier-derived default. + pub model_override: Option, + /// The company's task board, so a [`TaskDispatched`] cycle can load the + /// dispatched card and write its result back. `None` off the task path (the + /// chat brain leaves the board untouched). + /// + /// [`TaskDispatched`]: crate::ports::types::CompanyEvent::TaskDispatched + pub tasks: Option>, + /// The company's artifact store, so a dispatched card's output is recorded + /// as a versioned artifact (#187) instead of only as note text. `None` + /// leaves the board's behaviour exactly as before — the note is still + /// written either way, so an unwired artifact store loses nothing that + /// existed previously. + pub artifacts: Option>, + /// The company's ledgers, so an agent can read what has already been + /// decided, goaled or ruled out, record what it decides, and declare an axis + /// nobody anticipated. `None` builds no ledger tools at all — which is right + /// for a path with no company behind it, and is what every construction site + /// that predates them does. + pub ledgers: Option>, + /// The company's ledgers as they stood when the agent was built, for the + /// prompt catalogue. + /// + /// Resolved to **data** before deps construction because `build_agent` is + /// synchronous, the same shape the MCP servers already take. A ledger + /// declared mid-run is therefore reachable by every tool immediately (the + /// `ledger` argument is checked against the live registry at call time) and + /// appears in the *prompt* only from the next build — which is the honest + /// limit: system prompts are assembled once, and nothing can retroactively + /// edit one already in flight. + pub ledger_registry: crate::ledger::Registry, + /// The company's skill-delta store, so a built agent can see its effective + /// skill set (company-dir skills ∪ operator deltas ∪ custom docs) as read + /// tools + a prompt catalogue. `None` leaves the agent skill-less (the chat + /// path off the skills seam builds no skill surface). + /// + /// See [`skills`](crate::harness::skills) — this is the read-only catalogue + /// slice; skill *execution* is deferred. + pub skills: Option>, + /// The company's source directory (`companies/`), whose `skills/` + /// subtree supplies the committed skill bundles unioned into the effective + /// set. `None` surfaces only the operator deltas. + pub skills_source_dir: Option, + /// The repo-level shared skill library (`skills/*/SKILL.md`), the same set + /// the console's registry tab browses. Used only to heal pre-fix registry + /// installs, whose stored snapshot is a one-line stub — see + /// [`EffectiveSkills::materialize`](crate::harness::skills::EffectiveSkills::materialize). + /// Empty only when the host serves no shared skill library, where a stub + /// simply stays as it is; a platform-provisioned runtime otherwise receives + /// the library the application state loaded, same as the serve path. + pub skills_registry: Arc<[crate::company::SkillDoc]>, + /// The company's effective MCP servers (issue #50), resolved to **data** + /// (manifest `[[mcp_server]]` ∪ the runtime index, with each server's + /// outbound credential materialized to + /// [`AuthMaterial`](crate::company::mcp::AuthMaterial)) before deps + /// construction. `build_agent` is synchronous but the + /// [`SecretStore`](crate::ports::SecretStore) is async, so the runtime + /// builder resolves these ahead of time; each agent then filters the set by + /// its `mcp:*` tool grants. Empty leaves the agent with no MCP bridge tools. + pub mcp_servers: Vec, + /// Install-wide default MCP servers (issue #527), carried so the live + /// re-resolution in [`Harness::resolve_effective_mcp`] merges the same three + /// layers the boot-time resolution did. Without it a console edit would + /// re-resolve to manifest ∪ runtime and silently drop every default. + pub default_mcp_servers: Vec, + /// The company's durable [`FactStore`], surfaced to the orchestrator agent + /// through the `query_company` read tool (issue #53). `None` leaves the + /// orchestrator without the facts half of its insight surface (the chat path + /// off the orchestrator seam wires nothing). + pub facts: Option>, + /// The company's [`EventLog`], surfaced to the orchestrator agent through + /// the `query_company` read tool for recent-activity context (issue #53). + /// `None` leaves the orchestrator without the recent-events half. + pub events: Option>, + /// The shared delegation queue the orchestrator's `spawn_task` / + /// `delegate_to_desk` tools push onto and the [`HarnessBrain`] drains after + /// an orchestrator turn (issue #53). A [`DelegationQueue`] is a cheap shared + /// handle; cloning `HarnessDeps` shares one queue between the tools built + /// into the agent and the brain that drains it. Default is an empty queue. + pub delegations: DelegationQueue, + /// The shared handle to the company's [`WorkflowRunner`](crate::ports::WorkflowRunner), + /// so the orchestrator's `run_workflow` tool can reach the runner that is + /// itself built *from* these deps (issue #67). The runtime builder threads an + /// empty handle here, builds the [`HarnessWorkflowRunner`](crate::workflows::HarnessWorkflowRunner) + /// from a deps clone, then fills the shared cell — so the orchestrator agent + /// (built later from a clone of these deps) reaches it at turn time. The cell + /// holds a [`Weak`](std::sync::Weak), so deps↔runner is not a strong cycle. + /// Default (and any build with no runner) leaves it empty and the tool + /// reports workflow execution is not wired. + pub workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle, + /// The shared MCP failure queue the `OcMcpCallTool` decorator pushes onto and + /// the [`HarnessBrain`] drains after a turn (the error-hardening cell). Same + /// cheap-shared-handle pattern as [`Self::delegations`]; every string it + /// carries is scrubbed at the source. Default is an empty queue. + pub mcp_failures: McpFailureQueue, + /// The shared publish queue the `publish_artifact` tool stages onto and the + /// [`HarnessBrain`] drains at the end of a dispatch (issue #244). Same + /// cheap-shared-handle pattern as [`Self::mcp_failures`], and for the same + /// structural reason: tools are built **once per agent** while the card + /// varies **per dispatch**, so a tool cannot hold a task id or a store and + /// has to hand its work to something that does. + /// + /// Default is an empty queue, which simply means nothing is ever published + /// — every path degrades to "this task produced no artifact", which is a + /// legitimate outcome rather than a failure. + pub pending_publishes: crate::harness::publish::PendingPublishQueue, + /// The shared queue the orchestrator's `run_workflow` / `create_workflow` + /// tools stage a workflow reference onto and the [`HarnessBrain`] drains at + /// the end of a dispatch (issue #339) — the workflow half of a card's + /// output link. + /// + /// Same cheap-shared-handle pattern as [`Self::pending_publishes`], and for + /// the same structural reason: the tools are built **once per agent** while + /// the card varies **per dispatch**, so a tool cannot hold a task id and has + /// to hand its work to something that does. + /// + /// Default is an empty queue, which simply means no card ever links to a + /// workflow — the stamp falls back to the attempt's trace, which is a + /// complete answer rather than a missing one. + pub workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue, + /// The bounded, in-process cache the orchestrator's `run_workflow` tool fills + /// with each successful run's node output and the `read_run_output` companion + /// reads back (issue #418) — so a preview the run summary clipped is + /// reachable within the same turn. + /// + /// Same cheap-shared-handle pattern as [`Self::workflow_refs`]: the run tool + /// that stores and the read tool that serves are built in one `build_agent` + /// pass off the same deps clone, so they share one cache. Default is an empty + /// cache; nothing durable rides on it (the console run drawer is the durable + /// record), so a fresh process simply starts with nothing to read back. + pub run_outputs: crate::harness::orchestrator::RunOutputCache, + /// The DURABLE, console-facing per-node run output store (issue #596) — + /// distinct from [`Self::run_outputs`] above, which is the in-process, + /// evictable agent cache. The workflow runner persists each settled run's + /// bounded node output here so a *past* run reopened from History shows what + /// every node produced. `None` (the default build, and every unwired test) + /// degrades the persist to a no-op, exactly like [`Self::events`]. + pub run_output_store: Option>, + /// Issue #274's per-workflow snapshot ring, so the orchestrator's + /// `update_workflow` / `delete_workflow` tools (issue #661, M7) write + /// through the same undo-and-cascade path the console's `PUT`/`DELETE` + /// routes do — an agent edit is recoverable on exactly the terms an + /// operator's is. + /// + /// `None` (the default build, and every unwired test) makes those two tools + /// refuse rather than degrade, unlike [`Self::events`]. The asymmetry is the + /// point: a missing journal loses an audit line, while a missing revision + /// store loses the only copy of the graph being overwritten. + pub workflow_revisions: Option>, + /// The shared approval-request queue every agent's [`ApprovalPolicy`] pushes + /// a `RequireApproval` decision onto and the [`HarnessBrain`] drains after a + /// turn, parking each request through + /// [`CycleHost::park_effect`](crate::ports::brain::CycleHost::park_effect) + /// so it reaches the operator's Approvals page (issue #172). Same + /// cheap-shared-handle pattern as [`Self::delegations`]; the default is an + /// empty queue, which simply means nothing is ever parked. + pub approval_requests: ApprovalRequestQueue, + /// The company's [`SecretStore`], so [`HarnessPool::ensure`] can **re-resolve** + /// the effective MCP server set on each call and rebuild the roster when a + /// console add/remove/enable-toggle changes it — the MCP-freshness fix (a + /// runtime-added server reaches the agent on its next turn, no restart). + /// `None` (default/tests) keeps the boot-resolved [`Self::mcp_servers`] + /// static, exactly as before. + pub secrets: Option>, + /// The per-company SSRF allowlist for the `web` toolbelt (Cell A), from the + /// manifest `[tools].web_allowed_domains`. Empty (the default) is *open + /// mode* — all public hosts allowed — while OpenHuman's upstream `url_guard` + /// still rejects private/loopback/link-local/metadata IPs regardless. A + /// non-empty list is strict (only those hosts + subdomains); `"*"` is an + /// explicit allow-all-public wildcard. Threaded verbatim into + /// [`toolbelt::web_tools`](crate::harness::toolbelt::web_tools). + pub web_allowed_domains: Vec, + /// The capability-tier filter applied to each agent's assembled tool vector + /// (Cell A seam). [`AllowAll`](crate::harness::toolbelt::CapabilityFilter::AllowAll) + /// (the default) is identity. When [`Self::plan`] is set, + /// [`HarnessPool::ensure`] overwrites this per turn with the tenant's + /// resolved filter; when the plan is `None` this stays the no-plan + /// fallback/test override. + pub capabilities: toolbelt::CapabilityFilter, + /// The company's source directory (`companies/`), from which a + /// workflow's `sub_workflow` nodes resolve a child by `workflow_id` + /// (`workflows/.toml`). Distinct from + /// [`Self::skills_source_dir`](Self::skills_source_dir) so the two seams stay + /// independent even though both currently derive from the same `seed_dir`. + /// `None` (default/tests, and platform-provisioned tenants with nothing on + /// disk) keeps the loud `UnwiredResolver`, so a reached `sub_workflow` node + /// fails clearly instead of resolving nothing. + pub workflow_source_dir: Option, + /// The tenant's capability tier plan (issue #108). `None` (the default) + /// leaves gating **off** — byte-identical to Cell A, [`Self::capabilities`] + /// is used verbatim. When set, [`HarnessPool::ensure`] resolves a per-tenant, + /// per-period, fail-closed [`CapabilityFilter`](toolbelt::CapabilityFilter) + /// from the [`UsageMeter`] before each turn and installs it on the roster it + /// builds. Resolved from the manifest `[plan]` section by the runtime builder. + pub plan: Option, + /// The MANAGED media-generation backend (issue #109). `None` (the default at + /// every construction site) fails closed — no image/video tools are wired. + /// Only the production runtime builder sets it, from + /// [`media_backend_from_env`](crate::harness::provider::media_backend_from_env) + /// (env-only — never a tenant secret). When `Some` **and** a company + /// explicitly grants `media`, [`build::build_agent`] wires the + /// [`toolbelt::media_tools`]; a grant with no credential wires nothing and + /// warns. + pub media: Option, + /// The per-tenant Composio configuration (issue #110). `None` (the default + /// at every construction site) fails closed — no Composio tools are wired. + /// [`HarnessPool::ensure`] re-resolves it each turn (folded into the roster + /// fingerprint) so a console token set/rotate takes effect next turn with no + /// restart. Only wired when a company **explicitly** grants `composio` **and** + /// a credential can be obtained: the company's own token under + /// [`composio::TOKEN_KEY`](crate::harness::composio::TOKEN_KEY) if it has one, + /// else this instance's platform identity. With neither, no tools are wired — + /// never a borrowed identity. + pub composio: Option, + + /// The per-company Chargebee connection (issue #788). `None` (the default at + /// every construction site) fails closed — no billing tools are wired. + /// Resolved from that company's own secret store, never from the + /// environment: two companies on one host bill two different sites. + /// `HarnessPool::ensure` re-resolves it each turn, so a key set or rotated in + /// the console takes effect next turn with no restart. + #[cfg(feature = "chargebee")] + pub chargebee: Option, + + /// The per-company PayPal connection (issue #789). `None` fails closed — + /// no wallet tools are wired. Resolved from that company's own secret store + /// and re-resolved each turn, like `chargebee`. + #[cfg(feature = "paypal")] + pub paypal: Option, + + /// The per-company hosting connection. `None` (the default at every + /// construction site) fails closed — no hosting tools are wired. Resolved + /// from that company's own secret store and re-resolved each turn, like + /// `chargebee`: two companies on one host deploy to two different hosting + /// accounts, and a deployment publishes files to the internet under the + /// account's own name. + pub hosting: Option, + /// The MANAGED web-search backend (issue #238). `None` (the default at every + /// construction site but the production runtime builder) **fails closed** — + /// no `web_search` tool is wired and agents behave exactly as before. + /// + /// Set by the runtime builder from + /// [`search_backend_from_env`](crate::harness::provider::search_backend_from_env) + /// (env-only — never a tenant secret) with the company's + /// `[tools].search_daily_calls` cap applied. When `Some` **and** a company + /// **explicitly** grants `search` (never via `*`), [`build::build_agent`] + /// wires [`search::search_tools`]; a grant with no credential wires nothing + /// and warns, media's shape exactly. + /// + /// The handle carries the company's shared daily-call ledger, so cloning + /// these deps across a roster gives every agent of the company one budget + /// rather than one each. + pub search: Option, + /// Issue #111 — the shared registry of in-flight, steerable runs. The + /// [`HarnessBrain`] registers a dispatched task / desk delegation here before + /// running it (and installs the steer stop-hook over the slot's control), so + /// an operator can pause / cancel / redirect it mid-flight. The **same** + /// handle is threaded onto the [`CompanyRuntime`](crate::company::runtime::CompanyRuntime) + /// so the operator steer routes reach it. A cheap shared handle (like + /// [`delegations`](Self::delegations)); the default is an empty registry, + /// which simply lists nothing and rejects every steer as `not in flight`. + pub steer: crate::company::steer::InflightRegistry, + /// Issue #383 — the shared set of cancellable workflow runs. The + /// orchestrator's `run_workflow` tool mints its run context through this, so + /// an agent-initiated run appears in the same map the console's cancel route + /// reads and is stoppable like any other. The runtime builder threads in the + /// same handle it puts on the [`CompanyRuntime`](crate::company::CompanyRuntime); + /// the default is a private map nothing else can see, which simply means the + /// tool's runs are not cancellable. + pub run_supervisor: crate::runtime::RunSupervisor, + /// Issue #170 — the ports an `output` node's `destination` needs to route a + /// finished workflow's report to a person or a channel (mail handle, inbox, + /// user directory, wired channels), bundled so this struct grows one field + /// rather than four. + /// + /// Read post-engine by + /// [`deliver_outputs`](crate::workflows::delivery::deliver_outputs) — never + /// by the engine, which knows nothing about destinations. `None` (the + /// default at every construction site but the production runtime builder) + /// **fails closed and loud**: nothing is sent and the run result carries a + /// `failed` row saying delivery is not wired, so an authored destination can + /// never quietly do nothing. + pub delivery: Option, + /// Issue #237 — the company's shared workspace note tree, so agents can + /// read (and, under an explicit `workspace` grant, revise) the operator's + /// standards and playbooks instead of guessing at them. + /// + /// The same [`WorkspaceStore`](crate::ports::WorkspaceStore) handle the + /// console's REST/GraphQL surface writes through, so an operator edit is + /// visible to the next agent turn with no rebuild — the tools hold no + /// snapshot and hit the store per call. `None` (the default at every + /// construction site but the production runtime builder) **fails closed**: + /// no workspace tools are wired and agents behave exactly as before. + pub workspace: Option>, + /// Issue #245, agent half — the company's [`RepoManager`], so an agent that + /// explicitly grants `repo` can check a bound repository out and read a + /// pull request. `None` (the default at every construction site but the + /// production runtime builder) **fails closed**: no repository tools are + /// wired and agents behave exactly as before. + /// + /// [`RepoManager`]: crate::runtime::RepoManager + pub repos: Option>, + /// The company's bound repositories, resolved to **data** before deps + /// construction — the `mcp_servers` doctrine, and for the same reason: + /// [`build::build_agent`] is synchronous while reading the binding index is + /// async, and the tool descriptions name what is bound so a model does not + /// have to guess. Empty means nothing is bound, which is also what makes a + /// `repo` grant with no bindings wire nothing and warn. + pub repo_bindings: Vec, + /// The shared per-turn ledger of checkouts and diff spills, so the + /// [`CheckoutJanitor`](brain::CheckoutJanitor) claimed at each entry point + /// can delete them however the turn ends. + /// + /// Same cheap-shared-handle pattern as [`Self::pending_publishes`], and for + /// the same structural reason: the tools are built **once per agent** while + /// the deletion boundary is **per turn**. Default is an empty ledger, which + /// simply means nothing is ever recorded for deletion — the boot sweep is + /// the backstop. + pub checkouts: repo::CheckoutLedger, +} + +/// One live openhuman agent, keyed by its manifest id. +pub struct CompanyAgent { + /// The manifest agent id. + pub agent_id: String, + /// The manifest agent's human-readable role. + pub role: String, + /// This teammate's manifest `budget_usd_daily` cap, carried onto the roster + /// so the dispatch gate in [`HarnessPool::run_inner`] can read it without + /// re-loading the manifest per turn (issue #304). + /// + /// `None` for an uncapped teammate — and for every overlay teammate, which + /// carries no per-agent cap in v1. + pub budget_usd_daily: Option, + /// The embedded openhuman session. A [`Mutex`] because a `turn` takes + /// `&mut self` and one agent must serialise its own turns. + agent: Mutex, +} + +/// The graceful reply returned when a turn yields the transient empty-response +/// class twice — so chat never shows a bare "Couldn't send" for a model hiccup. +const GRACEFUL_EMPTY_REPLY: &str = "Sorry — I hit a temporary model hiccup and couldn't produce a reply. Please resend your message."; + +/// The operator-facing notice returned when the plan-level total token ceiling +/// (issue #188) is reached — a hard dispatch refusal, so no model call is made. +/// Surfaced as the turn's reply on every dispatch path (operator chat, task, +/// steered/background), since they all funnel through +/// [`HarnessPool::run_inner`](HarnessPool::run_inner). +const TOTAL_BUDGET_EXHAUSTED_NOTICE: &str = + "Token budget for this period is exhausted — dispatch paused until the period resets."; + +/// The operator-facing notice returned when one teammate has spent its manifest +/// `budget_usd_daily` (issue #304) — a hard dispatch refusal for that teammate +/// only, made before any model call. +/// +/// Deliberately a *visible refusal* rather than a silent no-op, and deliberately +/// per-teammate: the rest of the company keeps running, and the operator is told +/// which desk stopped, what its cap is, and when it comes back. There is no +/// per-call unit to park at turn level — an inference turn is not a tool call — +/// so a notice is the honest answer, mirroring +/// [`TOTAL_BUDGET_EXHAUSTED_NOTICE`]. +fn agent_budget_exhausted_notice(agent_id: &str, cap_usd: f64) -> String { + format!( + "{agent_id} has reached its daily spend cap of ${cap_usd:.2} — dispatch to this teammate \ + is paused until the cap resets at 00:00 UTC. Other teammates are unaffected." + ) +} + +/// The classification of a single `agent.turn` attempt, for the retry wrapper. +enum AttemptOutcome { + /// A non-empty reply. + Reply(String), + /// The transient empty-response class (an empty/blank reply, or the model's + /// "empty response" error) — retryable. + Empty, + /// A hard error (budget/auth/build/etc.) — propagated loudly, never swallowed. + Hard(OpenCompanyError), +} + +/// The result of a completed turn: the reply text plus the scrubbed +/// [`TurnStep`] timeline folded from the turn's progress stream. +/// +/// The steps are per-bubble: the operator bubble carries the orchestrator's +/// steps, a delegated desk bubble carries that desk lead's steps. They ride the +/// wire on [`OutboundMessage::steps`](crate::ports::types::OutboundMessage) and +/// are **never** written to memory ([`HarnessPool::run`] persists +/// `outcome.reply` only). +#[derive(Debug, Clone)] +pub struct TurnOutcome { + /// The agent's reply text. + pub reply: String, + /// The scrubbed, folded processing steps (empty for a memory-served or + /// tool-less turn — the zero-steps tell). + pub steps: Vec, + /// Whether this turn **paused at its tool-iteration cap** rather than + /// finishing what it set out to do (issue #926). + /// + /// A capped turn is not an error and never has been: openhuman stops the + /// tool loop, makes one extra tools-disabled call asking the model for a + /// resumable "Done so far / Next steps" checkpoint, and returns that as an + /// ordinary `Ok(reply)`. So the reply reads like a finished answer, and + /// nothing in the text, the steps or the error channel distinguishes "I + /// answered you" from "I ran out of steps mid-task" — which is exactly what + /// the operator could not tell. + /// + /// Read from openhuman's public + /// [`Agent::last_turn_hit_cap`](oh::agent::Agent::last_turn_hit_cap) while + /// the agent lock is still held, the same under-lock idiom + /// [`read_turn_usage`] uses. `false` on every path that returns an outcome + /// **without** running a model turn (the two pre-turn budget refusals, the + /// ACP fold) — a refusal is not a pause, and labelling one as a cap hit + /// would tell the operator to reply "continue" to a turn that never ran. + pub hit_iteration_cap: bool, + /// The in-turn **spend halt**, when one stopped this turn (issue #1032). + /// + /// `Some` exactly when the teammate declared a `budget_usd_daily`, the + /// [`SpendStopHook`](crate::harness::spend::SpendStopHook) armed for it + /// fired, and the turn therefore stopped short of the answer it was working + /// towards. `None` on every other path, including every turn by a teammate + /// who declared no budget — no hook is installed for them, so there is + /// nothing that could have halted them. + /// + /// A **separate** field from [`hit_iteration_cap`](Self::hit_iteration_cap) + /// rather than another reading of it, because the two are different + /// outcomes needing different operator actions: a step pause is resumable + /// with "continue", a spend halt means the work costs more than its budget + /// allows and asking again just spends more. #988 pinned that they are + /// distinguishable — a budget halt reads `last_turn_hit_cap() == false`, + /// because the run paused *below* `max_tool_iterations` — which is why this + /// could not be folded into the existing flag. + /// + /// Carries the figures rather than a bare `bool` so the notice can say what + /// was spent against which cap, and names the teammate so a chain of turns + /// cannot report a number the operator has no way to attribute. + pub halted_for_spend: Option, +} + +/// What one in-turn spend halt cost, and whose cap it was measured against +/// (issue #1032). +/// +/// The figures are the ones this crate already owns: the cap is +/// [`CompanyAgent::turn_spend_cap_usd`], and the spend is the sum of the +/// [`TurnUsage::cost_usd`](crate::harness::cost::TurnUsage::cost_usd) totals the +/// turn already reports. Deliberately **not** parsed out of the vendored hook's +/// `reason` string, which is a developer-facing trace line whose shape is +/// upstream's to change. +/// +/// `agent` is carried because one operator bubble can cover a responder turn, a +/// desk turn and a relay turn, each with its own cap. The iteration-cap notice +/// declines to name a number for exactly that reason; naming the teammate is +/// what makes a number attributable, and is why this one can be quoted where +/// that one could not. +#[derive(Debug, Clone, PartialEq)] +pub struct SpendHalt { + /// The teammate whose cap was reached. + pub agent: String, + /// What that teammate's turn had spent when the brake fired, in USD. + /// + /// Can exceed [`cap_usd`](Self::cap_usd): the brake fires *between* tool + /// iterations, so the call that crossed the line has already been paid for. + pub spent_usd: f64, + /// The cap it was measured against, in USD — the teammate's declared + /// `budget_usd_daily`. + pub cap_usd: f64, +} + +impl CompanyAgent { + /// Runs one turn against this agent, returning its reply text and the + /// per-attempt token/cost totals. + /// + /// **Empty-response hardening (the error-hardening cell)**: the hosted brain + /// occasionally returns a transient empty completion, which openhuman + /// surfaces as an error. Rather than letting the operator see a bare + /// "Couldn't send", this wrapper retries **once**; if the second attempt is + /// still empty it returns a graceful, scrubbed message instead of an `Err`. + /// **Non-transient** errors (budget, auth, build) still propagate loudly — no + /// blanket swallow. Every attempt's usage is returned so the cost hook meters + /// what the model actually consumed (a burnt empty attempt still costs + /// tokens). + /// + /// The usage is read from each just-completed turn via openhuman's public + /// [`Agent::last_turn_usage`](oh::agent::Agent::last_turn_usage) accessor + /// while the agent lock is still held. An offline provider that reports no + /// usage yields a zero [`TurnUsage`], which the cost hook treats as inert. + /// + /// **Activity-trace**: this is the one site holding `&mut Agent`, so it is + /// where the turn's [`AgentProgress`](oh::agent::progress::AgentProgress) + /// stream is captured. A per-turn `mpsc` channel is attached via + /// [`Agent::set_on_progress`](oh::agent::Agent::set_on_progress); an + /// always-draining collector task buffers every event so the turn loop never + /// blocks on a full channel; and after the turn (both attempts share the one + /// channel) the sink is detached, the collector joined, and the events folded + /// into the scrubbed [`TurnOutcome::steps`] by + /// [`steps::fold_steps`](crate::harness::steps::fold_steps). The sink is + /// per-turn *local* — deliberately not a [`HarnessDeps`] field — so parallel + /// turns never collide. + pub async fn run(&self, message: &str) -> crate::Result<(TurnOutcome, Vec)> { + self.run_with_steer(message, None, None, None).await + } + + /// Runs one turn with an optional operator **steer** control installed + /// (issue #111). + /// + /// When `steer` is `Some`, a [`SteerStopHook`](crate::harness::steer::SteerStopHook) + /// over the shared control is installed around the turn via + /// [`with_stop_hooks`](oh::agent::stop_hooks::with_stop_hooks). OpenHuman + /// fires stop hooks **between** tool-loop iterations (never mid-tool-call), + /// so an operator pause / cancel / redirect halts the turn gracefully at the + /// next iteration boundary. The control is `Box::pin`ned at the task-local + /// scope boundary to avoid the nested-scope stack-overflow trap. + /// + /// When a steer is pending after the first attempt yields the transient + /// empty-response class, the one-shot retry is **skipped** — a cancel (or + /// pause) issued before any text is produced must not silently restart the + /// work. With no steer this is byte-identical to the pre-#111 `run`. + /// + /// When `run_sink` is `Some`, the same collector also writes each step + /// through to the [`RunStore`](crate::ports::RunStore) as it arrives, so a + /// dispatched card's trace is durable *during* the run rather than only + /// after it (issue #242). The await lives in the collector task, never in + /// the model loop, so a slow store slows only trace persistence. `None` + /// (chat turns, workflow nodes, every test) is byte-identical to the prior + /// buffer-only behaviour. + pub async fn run_with_steer( + &self, + message: &str, + steer: Option<&SteerControl>, + stream: Option, + run_sink: Option>, + ) -> crate::Result<(TurnOutcome, Vec)> { + // Per-turn progress sink + an always-draining collector, so a burst of + // events never blocks the turn loop on a full channel. + // + // When `stream` is `Some`, the collector *tees* each event live onto the + // transient [`turn_stream`](crate::turn_stream) bus as it arrives — + // mirroring OpenHuman's `spawn_progress_bridge` — so the console renders + // the tool timeline while the turn is still running. The same events are + // still buffered and folded into the durable `TurnStep`s below, so the + // live view and the final reply timeline are byte-identical. With `None` + // (background turns, non-`openhuman` build) this is exactly the prior + // buffer-only behaviour. + let (tx, mut rx) = tokio::sync::mpsc::channel::(1024); + let collector = tokio::spawn(async move { + let mut events = Vec::new(); + let mut seq: u64 = 0; + // Mirrors `fold_steps`' thinking-run coalescing so the live timeline + // emits the same "Thinking" rows the final folded one does. + let mut thinking_open = false; + while let Some(event) = rx.recv().await { + if let Some(ctx) = &stream + && let Some(frame) = steps::stream_event_from(&event, seq, &mut thinking_open) + { + crate::turn_stream::publish( + &ctx.company, + frame + .with_agent(ctx.agent_id.clone()) + .with_chat(ctx.chat_id.clone()), + ); + seq += 1; + } + // Durable half (#242): persist the step before moving on, so a + // process killed mid-run keeps every step written so far. + if let Some(sink) = &run_sink { + sink.record(&event).await; + } + events.push(event); + } + events + }); + + let mut agent = self.agent.lock().await; + agent.set_on_progress(Some(tx)); + + // Two hooks, both fired by openhuman between tool-loop iterations: + // + // * the **steer** hook, only when an operator control is provided (#111); + // * the **budget** hook, only when this teammate declares a + // `budget_usd_daily` cap (#988) — the in-turn spend brake. A teammate + // with no declared budget gets no hook, which matches the vendored + // runtime's own posture: openhuman constructs `BudgetStopHook` nowhere + // and explicitly "never hard-stops a user-present turn that isn't + // actively burning a live budget". A turn that never outruns a real + // budget has nothing to protect it from, and a blanket magic number no + // operator can see or change would be worse than none. + // + // A budget halt and an iteration-cap pause are **different outcomes**, not + // two spellings of one: openhuman reports the cap through + // `Agent::last_turn_hit_cap`, which stays `false` for a hook-driven stop + // (the run paused below `max_tool_iterations`, so its cap predicate does + // not hold). Part 1 of #926 makes the cap pause operator-visible; it must + // not inherit budget halts. + let mut hooks: Vec> = Vec::new(); + if let Some(control) = steer { + hooks.push(Arc::new(crate::harness::steer::SteerStopHook::new( + control.clone(), + ))); + } + // Issue #1032: the budget hook is *wrapped* rather than pushed bare, so + // the halt survives the boundary. Upstream's `StopDecision::Stop` is + // consumed inside openhuman's tool loop, which returns the run's text as + // an ordinary `Ok(reply)`; `with_stop_hooks` hands back only the + // future's value; and `last_turn_hit_cap()` is `false` here by design. + // Without the wrapper there is nothing left to read, and a turn stopped + // for spend is indistinguishable from one that finished. + // + // The predicate itself stays upstream's — the wrapper only observes it. + let mut spend_brake: Option<(f64, Arc)> = None; + if let Some(cap) = self.turn_spend_cap_usd() { + let hook = crate::harness::spend::SpendStopHook::new(cap); + // Taken before the hook is boxed into the task-local list; once it + // is an `Arc` the concrete type is unreachable. + spend_brake = Some((cap, hook.halted())); + hooks.push(Arc::new(hook)); + } + + // `Box::pin` at the task-local scope boundary (the nested-scope + // stack-overflow trap). The turn body owns the retry classification and + // reports every attempt's usage. + let (reply, usages): (crate::Result, Vec) = + oh::agent::stop_hooks::with_stop_hooks( + hooks, + Box::pin(async { + let mut usages: Vec = Vec::new(); + let first = agent.turn(message).await; + usages.push(read_turn_usage(&agent)); + let reply: crate::Result = match self.classify_turn(first) { + AttemptOutcome::Reply(reply) => Ok(reply), + AttemptOutcome::Hard(err) => Err(err), + AttemptOutcome::Empty => { + // Retry-guard edge: skip the one-shot retry when an + // operator steer already pends, so a cancel/pause + // before any text can't restart the work. + // + // Issue #1032 adds the second guard, on the same + // reasoning: the work was stopped on purpose, and an + // empty reply is not licence to restart it. The + // retry is a fresh `agent.turn`, so openhuman builds + // it a fresh `TurnCost` — the brake's accumulator + // starts back at zero, and a teammate that had just + // exhausted its cap could spend up to a whole cap + // again before the hook fired a second time. The + // brake is armed per turn, so nothing else here + // would stop it. + // + // **Defence in depth, not a fix to an observed bug, + // and the difference is recorded so nobody re-derives + // it.** The `Empty` arm appears to be unreachable + // after a halt: a halt implies at least one completed + // tool iteration, and openhuman answers the post-halt + // wrap-up with its own synthesised "here's what I did + // this turn" summary — which it substitutes even when + // the wrap-up call returns blank text OR no choices + // at all. Both were scripted against the real turn + // loop and neither reached this arm, so there is no + // test here that would fail without this guard, and + // one was deliberately not left behind pretending + // otherwise. What the guard buys is that the + // invariant stops depending on that substitution + // staying true across a vendored bump. + // + // `halted_for_spend` below still reports the halt + // either way, so the operator gets the notice that + // explains a stub reply rather than silence. + let spend_halted = spend_brake.as_ref().is_some_and(|(_, halted)| { + halted.load(std::sync::atomic::Ordering::SeqCst) + }); + if steer.map(|c| c.requested()).unwrap_or(false) || spend_halted { + Ok(crate::harness::mcp_probe::scrub(GRACEFUL_EMPTY_REPLY, &[])) + } else { + let second = agent.turn(message).await; + usages.push(read_turn_usage(&agent)); + match self.classify_turn(second) { + AttemptOutcome::Reply(reply) => Ok(reply), + AttemptOutcome::Empty => Ok(crate::harness::mcp_probe::scrub( + GRACEFUL_EMPTY_REPLY, + &[], + )), + AttemptOutcome::Hard(err) => Err(err), + } + } + } + }; + (reply, usages) + }), + ) + .await; + + // Detach the sink (drops the only remaining `Sender`, closing the + // channel), release the agent lock, then drain + fold. A `Hard` error + // still runs this cleanup before propagating, so the collector never + // leaks. + agent.set_on_progress(None); + // Issue #926: read the cap flag while the lock is still held, the same + // under-lock idiom `read_turn_usage` uses above. Not draining, so the + // retry path's second attempt simply overwrites the first's value — + // which is right: the outcome describes the attempt that produced the + // reply being returned. + let hit_iteration_cap = agent.last_turn_hit_cap(); + drop(agent); + let events = collector.await.unwrap_or_default(); + // The cap openhuman was actually enforcing, for the trace only. Taken + // from the last `IterationStarted` rather than from config, so the log + // reports the number the turn ran under instead of the one this crate + // believes it configured. Deliberately NOT plumbed into the operator + // notice: one notice can cover a responder turn, a desk turn and a + // relay turn, and naming one of their caps would be a number the + // operator cannot map back to anything. + let iteration_cap = events.iter().rev().find_map(|event| match event { + oh::agent::progress::AgentProgress::IterationStarted { max_iterations, .. } => { + Some(*max_iterations) + } + _ => None, + }); + if hit_iteration_cap { + tracing::info!( + agent = %self.agent_id, + iteration_cap, + "[turn] paused at the tool-iteration cap; the reply is a resumable checkpoint, not a finished answer" + ); + } + // Issue #1032: read the spend brake the same way. Not under the agent + // lock — the flag lives on the hook, not on the vendored session, and + // the hook has already finished running by the time `with_stop_hooks` + // returns. + // + // The spend is summed over every attempt's usage rather than read from + // the hook, so the figure covers the retry path's second attempt too: + // both were paid for, and reporting only one would understate what the + // turn actually cost. + let halted_for_spend = spend_brake.and_then(|(cap_usd, halted)| { + halted + .load(std::sync::atomic::Ordering::SeqCst) + .then(|| SpendHalt { + agent: self.agent_id.clone(), + spent_usd: usages.iter().map(|usage| usage.cost_usd).sum(), + cap_usd, + }) + }); + if let Some(halt) = &halted_for_spend { + tracing::info!( + agent = %self.agent_id, + spent_usd = halt.spent_usd, + cap_usd = halt.cap_usd, + "[turn] halted at the in-turn spend cap; the reply stops short of the work it was doing" + ); + } + let steps = steps::fold_steps(events); + + let reply = reply?; + Ok(( + TurnOutcome { + reply, + steps, + hit_iteration_cap, + halted_for_spend, + }, + usages, + )) + } + + /// This turn's in-turn spend ceiling, in USD — the value that + /// [`BudgetStopHook`](oh::agent::stop_hooks::BudgetStopHook) halts the turn + /// at, armed only when the teammate declares a `budget_usd_daily` cap + /// (issue #988). `None` means no hook is installed. + /// + /// This mirrors the vendored runtime's own posture. OpenCompany's plan-level + /// token ceiling and a teammate's `budget_usd_daily` are **pre-dispatch** — + /// they decide whether to start a turn and cannot see inside one — and + /// openhuman itself constructs `BudgetStopHook` nowhere, applying only an + /// opt-in token-based goal hook. So this crate, like upstream, arms the + /// in-turn brake only for a teammate who has opted into a budget: a declared + /// `budget_usd_daily` cap also bounds any single turn of that teammate's, so + /// the worst-case overshoot is "one daily cap" rather than "one turn, of + /// unknown size". A teammate with no declared budget gets no hook — the + /// runtime never hard-stops a turn that isn't actively burning a live budget + /// — and there is no blanket magic number no operator can see or change. + /// + /// A non-finite or non-positive manifest value is ignored (no hook armed) + /// rather than forwarded: the vendored hook fails closed on a malformed cap + /// and would halt every turn at iteration one. Such a teammate is already + /// refused before dispatch (`spent >= cap` holds at zero spend), so this only + /// guards the path where no meter was available to make that call. + fn turn_spend_cap_usd(&self) -> Option { + match self.budget_usd_daily { + Some(daily) if daily.is_finite() && daily > 0.0 => Some(daily), + _ => None, + } + } + + /// Classify one `agent.turn` result for the retry wrapper. + fn classify_turn(&self, result: anyhow::Result) -> AttemptOutcome { + match result { + Ok(reply) if reply.trim().is_empty() => AttemptOutcome::Empty, + Ok(reply) => AttemptOutcome::Reply(reply), + Err(err) if is_transient_empty_response(&err) => AttemptOutcome::Empty, + Err(err) => AttemptOutcome::Hard(OpenCompanyError::Harness(format!( + "turn for '{}': {err}", + self.agent_id + ))), + } + } +} + +/// Reads the just-completed turn's usage (zero when the provider reported none). +fn read_turn_usage(agent: &Agent) -> TurnUsage { + agent + .last_turn_usage() + .map(|u| TurnUsage { + input_tokens: u.input_tokens, + output_tokens: u.output_tokens, + cached_input_tokens: u.cached_input_tokens, + cost_usd: u.cost_usd, + }) + .unwrap_or_default() +} + +/// Whether a turn error is the transient empty-response class openhuman raises +/// instead of a silent blank reply. Matched on the error chain's message +/// (`turn` returns `anyhow::Result`, so the typed `AgentError` is erased): +/// "The model returned an empty response…". +fn is_transient_empty_response(err: &anyhow::Error) -> bool { + format!("{err:#}") + .to_ascii_lowercase() + .contains("empty response") +} + +/// What a workspace-ensure attempt should say, given what the last attempt for +/// the same agent said (issue #449). +/// +/// The attempt itself is per dispatch and stays that way — see +/// [`note_workspace_attempt`](HarnessPool::note_workspace_attempt) for why +/// memoising it is the wrong fix. Only the *reporting* is edge-triggered. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum WorkspaceReport { + /// The first failure since this agent was last healthy: report it. + Failed, + /// Still failing, and already reported: say nothing. + StillFailing, + /// Working again after a reported failure: say so once, so a reader who saw + /// the error learns it ended. + Recovered, + /// Working, and was already working: say nothing. + StillHealthy, +} + +impl WorkspaceReport { + /// Whether this transition has anything to log at all. + pub(crate) fn is_silent(self) -> bool { + matches!(self, Self::StillFailing | Self::StillHealthy) + } +} + +/// Folds one attempt's outcome into the set of currently-failing keys and +/// returns what to report. +/// +/// Pure but for the `failing` set it edits, so the whole state machine is +/// testable without a model, a roster or a filesystem. `failing` holds exactly +/// the keys whose last attempt failed **and** whose failure has been reported; +/// `failed` is this attempt's outcome. +fn workspace_report(failing: &mut HashSet, key: &K, failed: bool) -> WorkspaceReport +where + K: std::hash::Hash + Eq + Clone, +{ + if failed { + // `insert` returns false when the key was already there — i.e. the + // previous attempt failed and was already reported. + if failing.insert(key.clone()) { + WorkspaceReport::Failed + } else { + WorkspaceReport::StillFailing + } + } else if failing.remove(key) { + WorkspaceReport::Recovered + } else { + WorkspaceReport::StillHealthy + } +} + +/// A pool of live agents, one roster per company. +pub struct HarnessPool { + agents: RwLock>>>, + /// Fingerprint of the effective MCP server set the cached roster was built + /// from, keyed by company. Drives MCP-freshness: [`ensure`](Self::ensure) + /// rebuilds the roster whenever the fingerprint changes. + mcp_fingerprints: RwLock>, + /// Fingerprint of the overlay-agent set (issue #71 — Active Runtime + /// Teammates) the cached roster was built from, keyed by company. Drives + /// overlay-agent freshness: [`ensure`](Self::ensure) rebuilds the roster + /// whenever an operator- or orchestrator-added teammate is added/removed, + /// mirroring the MCP-freshness fingerprint above. + overlay_fingerprints: RwLock>, + /// Fingerprint of the resolved [`CapabilityFilter`](toolbelt::CapabilityFilter) + /// the cached roster was built from, keyed by company (issue #108). Drives + /// capability-budget freshness: [`ensure`](Self::ensure) re-resolves the + /// tenant's filter from the [`UsageMeter`] on every call and rebuilds the + /// roster whenever the denied-namespace set changes — so a tier that crosses + /// its token budget switches off on the company's **next** turn. With no + /// plan ([`HarnessDeps::plan`] `None`) the filter is the static + /// [`HarnessDeps::capabilities`], whose fingerprint never moves — no rebuild, + /// byte-identical to Cell A. + capability_fingerprints: RwLock>, + /// Fingerprint of the resolved per-tenant [`TenantComposio`](composio::TenantComposio) + /// config the cached roster was built from, keyed by company (issue #110). + /// Drives Composio-freshness: [`ensure`](Self::ensure) re-resolves the token + /// (+ toolkit allowlist) from the [`SecretStore`] on every call and rebuilds + /// the roster whenever it changes — so a console token set/rotate/clear takes + /// effect on the company's **next** turn with no restart. With no secret + /// store wired the config is the static [`HarnessDeps::composio`], whose + /// fingerprint never moves. + composio_fingerprints: RwLock>, + /// Fingerprint of the billing connections (Chargebee #788, PayPal #789) the + /// cached roster was built from, keyed by company. + /// + /// Without this axis a credential saved from the console reaches nothing + /// until a restart — the roster is cached, so `build_agent` is never called + /// again to notice it. That was live for both integrations until the tools + /// were observed missing from an agent whose settings page said "Connected". + billing_fingerprints: RwLock>, + /// Fingerprint of the company's bound-repository set the cached roster was + /// built from, keyed by company (issue #245). Drives repository freshness: + /// [`ensure`](Self::ensure) re-reads the binding index from the + /// [`SecretStore`] on every call and rebuilds the roster whenever it moves — + /// so a bind, a credential rotation and a revoke each reach the agent on the + /// company's **next** turn with no restart. + /// + /// All three have to move it, which is why the fingerprint is over + /// `(key, token_fingerprint, branches)` rather than over the key alone: a + /// rotation changes nothing about *which* repositories exist, and a roster + /// that kept a tool description naming a binding whose credential has since + /// been revoked would offer an agent a checkout that can no longer fetch. + /// With no secret store wired the set is the static + /// [`HarnessDeps::repo_bindings`], whose fingerprint never moves. + repo_fingerprints: RwLock>, + /// Fingerprint of the operator skill-delta set the cached roster was built + /// from, keyed by company (issue #41). Drives skill-delta freshness: + /// [`ensure`](Self::ensure) re-fetches the deltas from the + /// [`SkillStateStore`](crate::ports::skills_state::SkillStateStore) on every + /// call and rebuilds the roster whenever they change — so a skill + /// authored / edited / enabled / disabled in the console Skills tab reaches + /// the agent on the company's **next** turn with no restart. Without this + /// axis the four fingerprints above are all stable on a skills-only change, + /// the fast path returns early, and the new skill never surfaces until a + /// process restart (the regression this fixes). With no skill store wired + /// the delta set is always empty — stable fingerprint, no rebuild. + skill_fingerprints: RwLock>, + /// Fingerprint of the operator budget-override set the cached roster was + /// built from, keyed by company (issue #343). Drives budget freshness: + /// [`ensure`](Self::ensure) re-resolves the overrides from + /// [`HarnessDeps::store`] on every call and rebuilds the roster whenever a + /// cap is set, changed, cleared or reset — so a budget edited on the console + /// Team page reaches the dispatch gate and the per-agent + /// [`ApprovalPolicy`](policy::ApprovalPolicy) on the company's **next** turn, + /// with no restart and no redeploy. That is the entire point of #343: the + /// cap is enforced from the roster, and without this axis every other + /// fingerprint is stable on a budget-only change, so the fast path would + /// reuse a roster still carrying the old cap until the process restarted. + /// A company that never sets an override keeps an empty set and a stable + /// fingerprint — no rebuild, byte-identical to the pre-#343 behaviour. + budget_fingerprints: RwLock>, + /// Per-company fingerprint of the operator `[policy]` override (issue #562), + /// so a console tier change rebuilds the roster instead of waiting for a + /// restart. Without this axis the override persists and is silently ignored: + /// `ApprovalPolicy` is built once per roster, not once per call. + policy_fingerprints: RwLock>, + /// Per-company fingerprint of the desk scoping a roster's grants resolve + /// through — which desks exist, who sits on them, and each one's tool + /// ceiling. + /// + /// Needed for the same reason as [`Self::budget_fingerprints`]: a tool belt + /// is wired once per roster, not once per call, so without this axis a + /// console desk-ceiling edit (or seating a teammate on a restricted desk) + /// would leave every other fingerprint stable and the fast path would keep + /// serving the old belt until the process restarted. A company whose desks + /// declare no ceilings keeps a stable fingerprint and never rebuilds on this + /// axis. + desk_fingerprints: RwLock>, + /// Per-company fingerprint of the routed workspace documents — hashed over + /// their **bodies**, not merely their names. + /// + /// A persona is assembled once per roster, so without this axis an operator + /// editing a routed note would leave every other fingerprint stable and the + /// fast path would keep serving a prompt quoting the old text until the + /// process restarted. Hashing the names alone would have exactly that bug, + /// since the routing table does not move when a document's contents do — + /// which is the whole reason the routing layer is worth having. + /// + /// A company with no workspace store wired, or whose roles route nothing, + /// keeps a stable fingerprint and never rebuilds on this axis. + context_fingerprints: RwLock>, + /// The `(company, agent)` pairs whose last workspace-ensure failed and whose + /// failure has already been reported (issue #449). + /// + /// Not a memo of the *attempt* — see + /// [`note_workspace_attempt`](Self::note_workspace_attempt). Purely a record + /// of what has already been said, so an unmountable volume produces one + /// error line instead of one per turn forever. + /// + /// A `std::sync::Mutex` rather than a `tokio::sync::RwLock` like its + /// neighbours: the critical section is a single hash lookup with no `await` + /// in it, so the async lock would buy nothing and cost a scheduling point on + /// the dispatch path. + workspace_failures: std::sync::Mutex>, +} + +impl Default for HarnessPool { + fn default() -> Self { + Self::new() + } +} + +/// Whether a turn tees its progress onto the live [`turn_stream`](crate::turn_stream) +/// bus, and if so which chat thread its frames route to. `Off` for a turn with no +/// operator chat bubble (a dispatched task card or workflow agent node) — those +/// frames would misattribute to whatever thread most recently sent, so they +/// publish nothing (#125 review). `On { chat_id }` streams; `chat_id` is the +/// thread the durable reply is journaled under (`AgentReply.chat_id`), falling +/// back to the default desk when the caller addressed none. +#[derive(Clone, Copy)] +enum LiveStream<'a> { + Off, + On { chat_id: Option<&'a str> }, +} + +impl HarnessPool { + /// Builds an empty pool. + pub fn new() -> Self { + Self { + agents: RwLock::new(HashMap::new()), + mcp_fingerprints: RwLock::new(HashMap::new()), + overlay_fingerprints: RwLock::new(HashMap::new()), + capability_fingerprints: RwLock::new(HashMap::new()), + composio_fingerprints: RwLock::new(HashMap::new()), + billing_fingerprints: RwLock::new(HashMap::new()), + repo_fingerprints: RwLock::new(HashMap::new()), + skill_fingerprints: RwLock::new(HashMap::new()), + budget_fingerprints: RwLock::new(HashMap::new()), + policy_fingerprints: RwLock::new(HashMap::new()), + desk_fingerprints: RwLock::new(HashMap::new()), + context_fingerprints: RwLock::new(HashMap::new()), + workspace_failures: std::sync::Mutex::new(HashSet::new()), + } + } + + /// Records one workspace-ensure outcome for `(company, agent)` and returns + /// what it should say. + /// + /// **The attempt stays per dispatch.** The obvious fix for a repeating log + /// line — remember that this agent's workspace was already handled and stop + /// trying — is the wrong one in both directions, and this is why the + /// suppression is on the reporting rather than on the work: + /// + /// * Memoising **success** means a data dir wiped or restored *after* the + /// first successful turn is never noticed again, and every relative file + /// write is refused for the life of the process — the exact regression + /// issue #409 added the per-dispatch retry to prevent. + /// * Memoising **failure** means a volume that mounts a second late never + /// recovers, because nothing ever tries again. + /// + /// Both trade a noisy log for a broken agent. The retry is cheap (two + /// syscalls on the already-exists path, against a turn about to call a + /// model) and it is what makes the condition self-healing, so it keeps + /// running every time. What changes is that a persistent failure is stated + /// once rather than once per turn. + fn note_workspace_attempt( + &self, + company: &CompanyId, + agent_id: &str, + failed: bool, + ) -> WorkspaceReport { + let key = (company.clone(), agent_id.to_string()); + let mut failing = self + .workspace_failures + .lock() + .expect("workspace-failure set poisoned"); + workspace_report(&mut failing, &key, failed) + } + + /// Ensures a company's roster is built and cached. + /// + /// **MCP-freshness (the error-hardening cell)**: on every call, the effective + /// MCP server set is re-resolved (from the [`SecretStore`] when + /// [`HarnessDeps::secrets`] is wired, else the boot-resolved + /// [`HarnessDeps::mcp_servers`]) and fingerprinted. The roster is rebuilt when + /// it is absent **or** the fingerprint changed — so a console MCP + /// add/remove/enable-toggle reaches the agent on its **next turn**, with no + /// company restart (the "Parallel Search / BrowserBase" bug). When nothing + /// changed, the cached roster is reused (the common fast path), exactly as + /// before. + /// + /// **Overlay-agent freshness (issue #71)**: the live overlay-agent set is + /// re-resolved and fingerprinted the same way, from [`HarnessDeps::store`] + /// rather than the (possibly stale) `company` snapshot passed in — so a + /// teammate added through the console `POST .../team` route or the + /// orchestrator's `add_agent` tool becomes a real, addressable roster agent + /// on the company's **next** `ensure` call, with no restart. + /// + /// **Skill-delta freshness (issue #41)**: the operator skill deltas are + /// fetched from [`HarnessDeps::skills`] and fingerprinted **before** the + /// fast-path staleness check (not after it, as they were — the regression), + /// so a skill authored / edited / enabled / disabled in the console Skills + /// tab rebuilds the roster and reaches the agent on its **next** turn, even + /// when every other axis (MCP, overlay, capability, composio) is unchanged. + /// With no skill store wired the delta set is empty and the fingerprint is + /// stable — no rebuild, exactly as before. + /// + /// **Budget freshness (issue #343)**: the operator's per-teammate daily + /// spend caps ride the same live [`HarnessDeps::store`] read as the overlay + /// agents and are fingerprinted alongside them, so a cap set, raised, + /// cleared or reset from the console Team page rebuilds the roster and is + /// enforced on the company's **next** dispatch. Nothing downstream had to + /// change for this: the L1 gate in [`Self::run`] reads + /// [`CompanyAgent::budget_usd_daily`] and the policy arm reads the + /// [`ApprovalPolicy`](policy::ApprovalPolicy) both roster-built here, so + /// rebuilding the roster *is* the enforcement update. That is what makes + /// "no restart, no redeploy" a property of the design rather than a claim. + pub async fn ensure(&self, company: &CompanyRecord, deps: &HarnessDeps) -> crate::Result<()> { + // Re-resolve + fingerprint the effective MCP set (cheap; no rebuild yet). + let effective_mcp = self.resolve_effective_mcp(company, deps).await; + let mcp_fp = mcp_fingerprint(&effective_mcp); + + // Re-resolve + fingerprint the live overlay-agent set the same way, and + // the operator budget overrides riding the same store read (issue #343). + let overlay = self.resolve_effective_overlay(company, deps).await; + let overlay_fp = overlay_fingerprint(&overlay.agents); + let budget_fp = budget_fingerprint(&overlay.budgets); + let policy_fp = policy_fingerprint(overlay.policy.as_ref()); + // Desk scoping now decides capability (the middle level of the + // three-level narrowing), so it joins the staleness check: without this + // a console desk-ceiling edit — or seating a teammate on a restricted + // desk — would not reach the roster until a restart. + let desk_fp = + desk_scope_fingerprint(&overlay.desks, &overlay.desk_members, &overlay.desk_tools); + + // Re-resolve + fingerprint the tenant's capability filter (issue #108): + // a per-tenant, per-period, fail-closed budget read from the meter. With + // no plan this is the static `deps.capabilities`, whose fingerprint is + // stable — so a no-plan company never rebuilds on this axis. + let capability_filter = self.resolve_capability_filter(company, deps).await; + let capability_fp = capability_budget::filter_fingerprint(&capability_filter); + + // Re-resolve + fingerprint the per-tenant Composio config (issue #110): + // the token (+ toolkit allowlist) read live from the secret store, so a + // console token set/rotate/clear takes effect on the next turn. With no + // secret store wired this is the static `deps.composio`, whose + // fingerprint is stable — so that company never rebuilds on this axis. + let composio_config = self.resolve_composio(company, deps).await; + let composio_fp = composio::TenantComposio::fingerprint(&composio_config); + + // Re-resolve + fingerprint the billing connections (#788, #789) for the + // same reason as Composio above: both are set from the console, so a + // roster that never re-reads them leaves an agent without billing tools + // on a company whose settings page reads "Connected". + #[cfg(feature = "chargebee")] + let chargebee_config = self.resolve_chargebee(company, deps).await; + #[cfg(feature = "paypal")] + let paypal_config = self.resolve_paypal(company, deps).await; + // The hosting credential is set from the same settings surface and goes + // stale the same way, so it rides the same axis. + let hosting_config = self.resolve_hosting(company, deps).await; + // A build without either feature has no billing axis to go stale on, so + // the fingerprint is a constant and this company never rebuilds on it. + let billing_fp = { + use std::hash::Hasher; + // Always written to: the hosting axis below is ungated. + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + #[cfg(feature = "chargebee")] + hasher.write_u64(chargebee::TenantChargebee::fingerprint(&chargebee_config)); + #[cfg(feature = "paypal")] + hasher.write_u64(paypal::TenantPaypal::fingerprint(&paypal_config)); + hasher.write_u64(hosting::TenantHosting::fingerprint(&hosting_config)); + hasher.finish() + }; + + // Re-read + fingerprint the company's bound repositories (issue #245): + // one index document, read live, so a bind / rotate / revoke reaches the + // agent on the next turn. Only companies that explicitly grant `repo` + // touch the store on this axis; everything else resolves to the static + // `deps.repo_bindings` (empty at every construction site but the + // production builder), whose fingerprint never moves. + let repo_bindings = self.resolve_repo_bindings(company, deps).await; + let repo_fp = repo_binding_fingerprint(&repo_bindings); + + // Re-fetch + fingerprint the operator skill deltas (issue #41) BEFORE the + // fast-path check. A skills-only change leaves every other axis stable, so + // unless skills participate in the staleness check the cached roster is + // wrongly reused and a console-authored / edited / disabled skill never + // surfaces until a restart (the regression). `build_roster`/`build_agent` + // stay synchronous and fold these deltas into each agent's effective + // skill set; the same Vec is reused for the rebuild below (no re-fetch). + let mut skill_deltas = match &deps.skills { + Some(store) => store.list(&company.id).await?, + None => Vec::new(), + }; + // `[globals].disable = ["skill:…"]` reaches the effective set as a + // synthesized disabling delta rather than a second opt-out mechanism + // inside `EffectiveSkills`: the manifest and the console are then saying + // the same thing in the same vocabulary, and a disable always beats an + // enable there, so the company's own declaration wins over a console + // re-enable of a skill it opted out of. + skill_deltas.extend(globals_skill_disables(&company.manifest.globals.disable)); + let skill_deltas = skill_deltas; + let skill_fp = skill_delta_fingerprint(&skill_deltas); + + // Resolve the routed workspace documents (context routing) before the + // fast-path check, and fingerprint their *content*. Both halves matter: + // resolving here is what lets the synchronous `build_agent` fold them + // into a persona at all, and hashing the bodies rather than the file + // names is what makes an operator's edit to a routed note rebuild the + // roster. A name-only hash would leave an edited note invisible until a + // restart — the same staleness bug `skill_fp` above exists to close. + let routed_context = self + .resolve_routed_context(company, deps, &overlay.agents) + .await; + let context_fp = routed_context_fingerprint(&routed_context); + + { + let agents = self.agents.read().await; + let mcp_fingerprints = self.mcp_fingerprints.read().await; + let overlay_fingerprints = self.overlay_fingerprints.read().await; + let capability_fingerprints = self.capability_fingerprints.read().await; + let composio_fingerprints = self.composio_fingerprints.read().await; + let billing_fingerprints = self.billing_fingerprints.read().await; + let repo_fingerprints = self.repo_fingerprints.read().await; + let skill_fingerprints = self.skill_fingerprints.read().await; + let budget_fingerprints = self.budget_fingerprints.read().await; + let policy_fingerprints = self.policy_fingerprints.read().await; + let desk_fingerprints = self.desk_fingerprints.read().await; + let context_fingerprints = self.context_fingerprints.read().await; + if agents.contains_key(&company.id) + && mcp_fingerprints.get(&company.id) == Some(&mcp_fp) + && overlay_fingerprints.get(&company.id) == Some(&overlay_fp) + && capability_fingerprints.get(&company.id) == Some(&capability_fp) + && composio_fingerprints.get(&company.id) == Some(&composio_fp) + && billing_fingerprints.get(&company.id) == Some(&billing_fp) + && repo_fingerprints.get(&company.id) == Some(&repo_fp) + && skill_fingerprints.get(&company.id) == Some(&skill_fp) + && budget_fingerprints.get(&company.id) == Some(&budget_fp) + && policy_fingerprints.get(&company.id) == Some(&policy_fp) + && desk_fingerprints.get(&company.id) == Some(&desk_fp) + && context_fingerprints.get(&company.id) == Some(&context_fp) + { + return Ok(()); + } + } + + // Fold the freshly-resolved MCP set into the deps the roster is built + // from, so a changed set actually reaches the rebuilt agents. The clone + // shares every Arc / queue handle — only `mcp_servers` is overridden. + let mut fresh_deps = deps.clone(); + fresh_deps.mcp_servers = effective_mcp; + // Install the freshly-resolved capability filter on the deps the roster + // is built from, the same pattern as `mcp_servers` — so a tenant that + // crossed a tier budget gets a roster whose exec tools are actually + // trimmed. With no plan this is just `deps.capabilities` unchanged. + fresh_deps.capabilities = capability_filter; + // Install the freshly-resolved Composio config the same way, so a token + // set/rotate/clear reaches the rebuilt agents (issue #110). + fresh_deps.composio = composio_config; + #[cfg(feature = "chargebee")] + { + fresh_deps.chargebee = chargebee_config; + } + #[cfg(feature = "paypal")] + { + fresh_deps.paypal = paypal_config; + } + fresh_deps.hosting = hosting_config; + // And the freshly-read bindings (issue #245), so a repository bound or + // revoked in the console is what the rebuilt agents' tools resolve + // against — including the descriptions that name what is bound. + fresh_deps.repo_bindings = repo_bindings; + // Same treatment for the overlay-agent set: `company` may be a stale + // boot-time snapshot (e.g. `HarnessBrain::record`), so the roster is + // built from the live-resolved overlay set, not `company.overlay_agents`. + let mut fresh_company = company.clone(); + fresh_company.overlay_agents = overlay.agents; + // Same treatment for the budget overrides (issue #343): `build_roster` + // resolves every agent's cap through `fresh_company.effective_budget`, + // so installing the live set here is what carries a console budget edit + // into the roster the very next turn runs on. + fresh_company.overlay_budgets = overlay.budgets; + // The desk axis gets the same treatment, and needs it for the same + // reason: `build_roster` resolves every teammate's grants through + // `fresh_company.agent_desk_tools`, so the live desk set, seating and + // ceilings have to be the ones installed here. + fresh_company.overlay_desks = overlay.desks; + fresh_company.overlay_desk_members = overlay.desk_members; + fresh_company.overlay_desk_tools = overlay.desk_tools; + // Issue #562: same treatment for the policy override — `build_roster` + // resolves the tier through `fresh_company.effective_policy`, so installing + // the live value here is what carries a console tier change into the roster + // the next turn runs on. + fresh_company.overlay_policy = overlay.policy; + + // Issue #551 note — this rebuild deliberately touches no workspace. + // + // It used to provision `Agents//` for the roster it was about to + // build, because a teammate added at runtime (a manifest edit, the + // console's `add_member`, the orchestrator's `add_agent`) all land here + // as a moved overlay fingerprint and boot could not have known about + // them. That justification is gone: a member folder is no longer a + // function of the roster. `Agents/` and `Desks/` are laid down once at + // boot ([`RuntimeBuilder::build`]) and depend on nothing a rebuild can + // change, and `Agents//` is minted by + // [`ensure_agent_folder`](crate::company::workspace_scaffold::ensure_agent_folder) + // at the moment that agent first produces something — which is also the + // repair path if boot's create ever fail-softed, since the minter + // creates the root it needs. A rebuild-time call would now be a tree + // read that can only ever find its work already done. + let roster = build_roster(&fresh_company, &fresh_deps, &skill_deltas, &routed_context)?; + + let mut agents = self.agents.write().await; + agents.insert(company.id.clone(), roster); + self.mcp_fingerprints + .write() + .await + .insert(company.id.clone(), mcp_fp); + self.overlay_fingerprints + .write() + .await + .insert(company.id.clone(), overlay_fp); + self.capability_fingerprints + .write() + .await + .insert(company.id.clone(), capability_fp); + self.composio_fingerprints + .write() + .await + .insert(company.id.clone(), composio_fp); + self.billing_fingerprints + .write() + .await + .insert(company.id.clone(), billing_fp); + self.repo_fingerprints + .write() + .await + .insert(company.id.clone(), repo_fp); + self.skill_fingerprints + .write() + .await + .insert(company.id.clone(), skill_fp); + self.budget_fingerprints + .write() + .await + .insert(company.id.clone(), budget_fp); + self.policy_fingerprints + .write() + .await + .insert(company.id.clone(), policy_fp); + self.desk_fingerprints + .write() + .await + .insert(company.id.clone(), desk_fp); + self.context_fingerprints + .write() + .await + .insert(company.id.clone(), context_fp); + Ok(()) + } + + /// Re-resolves the company's capability filter (issue #108): with a plan + /// wired ([`HarnessDeps::plan`]), a per-tenant, per-period, fail-closed + /// budget read from the [`UsageMeter`] via + /// [`capability_budget::resolve_filter`]; without one, the static + /// [`HarnessDeps::capabilities`] verbatim (gating off). Never a boot + /// snapshot — resolved on every `ensure` so a tier switches off the turn + /// after its budget is crossed. + async fn resolve_capability_filter( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + ) -> toolbelt::CapabilityFilter { + match &deps.plan { + Some(plan) => { + capability_budget::resolve_filter( + plan, + deps.meter.as_deref(), + &company.id, + crate::ports::now_millis(), + ) + .await + } + None => deps.capabilities.clone(), + } + } + + /// Re-resolves the company's per-tenant Composio config (issue #110) from the + /// [`SecretStore`], so a console token set/rotate/clear takes effect on the + /// next turn. Only companies that **explicitly** grant `composio` touch the + /// secret store on this axis; others resolve to `None` (no tools). With no + /// secret store wired this degrades to the static [`HarnessDeps::composio`]. + /// + /// Resolution prefers the company's own stored token and falls back to this + /// instance's platform identity; with neither it yields `None` (fail closed). + /// Both the backend URL (from [`composio::COMPOSIO_BACKEND_URL_ENV`], then the + /// tenant API base [`composio::TINYHUMANS_API_URL_ENV`], then the prod + /// default) and the platform identity are read process-globally here, so a + /// live re-resolution keeps them even when nothing was stored at boot. + /// + /// Re-deriving the token source every turn costs nothing — building it reads + /// no file — and the roster that keeps it holds one instance for its whole + /// lifetime, so its rotation cache still works. + async fn resolve_composio( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + ) -> Option { + if !crate::company::grants_composio_explicit(&company.manifest.tools.allow) { + return None; + } + let toolkits = company.manifest.tools.composio.toolkits.clone(); + match &deps.secrets { + Some(secrets) => { + use crate::app::config::EnvSource; + let env = crate::app::config::ProcessEnv; + let url = env.get(composio::COMPOSIO_BACKEND_URL_ENV); + let api_url = env.get(composio::TINYHUMANS_API_URL_ENV); + composio::TenantComposio::resolve( + &company.id, + secrets.as_ref(), + toolkits, + url, + api_url, + crate::company::TinyhumansTokenSource::from_env(&env).map(std::sync::Arc::new), + ) + .await + } + None => deps.composio.clone(), + } + } + + /// Re-reads the company's Chargebee connection from the secret store, so a + /// key saved or rotated in Settings → Billing reaches the agent on its next + /// turn rather than at the next restart (issue #788). + /// + /// Only companies that **explicitly** grant `chargebee` read at all. With no + /// secret store wired this keeps the boot-resolved + /// [`HarnessDeps::chargebee`] — which was itself resolved from *this* + /// company's secret store by the runtime builder, so the fallback cannot + /// reach another tenant's credential. + /// + /// A transient **read error** keeps that connection too, with a warning, + /// rather than un-wiring the billing tools — the same direction + /// [`Self::resolve_repo_bindings`] and [`Self::resolve_effective_mcp`] + /// degrade in, and the safe one here for a specific reason: a stale + /// Chargebee credential is refused by Chargebee, which the agent surfaces as + /// a tool error it can report, whereas a tool that has vanished is invisible + /// to the agent — it simply stops being able to invoice and says nothing. + /// An absent credential still resolves to `None`; only the error case holds. + #[cfg(feature = "chargebee")] + async fn resolve_chargebee( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + ) -> Option { + if !crate::company::grants_chargebee_explicit(&company.manifest.tools.allow) { + return None; + } + let Some(secrets) = &deps.secrets else { + return deps.chargebee.clone(); + }; + match chargebee::TenantChargebee::resolve(secrets, &company.id).await { + Ok(resolved) => resolved, + Err(err) => { + tracing::warn!( + company = %company.id, + "[chargebee] could not read the billing credential; keeping the last known \ + connection: {err}" + ); + deps.chargebee.clone() + } + } + } + + /// The hosting equivalent, for the same reasons. + /// + /// Only companies that **explicitly** grant `hosting` read at all: a + /// deployment publishes a company's files to the public internet and can + /// provision a database it is billed for, so the catch-all `*` does not + /// confer it. + /// + /// A transient read error keeps the last known connection with a warning, + /// like `chargebee` and for the same reason: a stale hosting key is refused + /// by the provider, which the agent surfaces as a tool error it can report, + /// whereas a tool that has vanished is invisible to the agent — it simply + /// stops being able to deploy and says nothing. + async fn resolve_hosting( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + ) -> Option { + if !crate::company::grants_hosting_explicit(&company.manifest.tools.allow) { + return None; + } + let Some(secrets) = &deps.secrets else { + return deps.hosting.clone(); + }; + match hosting::TenantHosting::resolve(secrets, &company.id).await { + Ok(resolved) => resolved, + Err(err) => { + tracing::warn!( + company = %company.id, + "[hosting] could not read the hosting credential; keeping the last known \ + connection: {err}" + ); + deps.hosting.clone() + } + } + } + + /// The PayPal equivalent (issue #789), for the same reasons. + #[cfg(feature = "paypal")] + async fn resolve_paypal( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + ) -> Option { + if !crate::company::grants_paypal_explicit(&company.manifest.tools.allow) { + return None; + } + let Some(secrets) = &deps.secrets else { + return deps.paypal.clone(); + }; + match paypal::TenantPaypal::resolve(secrets, &company.id).await { + Ok(resolved) => resolved, + Err(err) => { + tracing::warn!( + company = %company.id, + "[paypal] could not read the billing credential; keeping the last known \ + connection: {err}" + ); + deps.paypal.clone() + } + } + } + + /// Re-reads the company's bound repositories (issue #245) from the + /// [`RepoManager`](crate::runtime::RepoManager), so a bind, a credential + /// rotation or a revoke reaches the roster on the next turn. + /// + /// Only companies that **explicitly** grant `repo` read at all; everything + /// else answers empty without touching the store, mirroring + /// [`Self::resolve_composio`]. A transient read error degrades to the + /// boot-resolved [`HarnessDeps::repo_bindings`] with a warning rather than + /// dropping an agent's repository tools mid-session — the same direction + /// [`Self::resolve_effective_mcp`] degrades in, and the safe one: a stale + /// binding list still resolves against real bindings, while an empty one + /// un-wires the tools entirely. + async fn resolve_repo_bindings( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + ) -> Vec { + if !crate::company::grants_repo_explicit(&company.manifest.tools.allow) { + return Vec::new(); + } + let Some(repos) = deps.repos.as_ref() else { + return deps.repo_bindings.clone(); + }; + match repos.list().await { + Ok(bindings) => bindings, + Err(err) => { + tracing::warn!( + company = %company.id, + "[repo] could not read the repository bindings; keeping the last known set: {err}" + ); + deps.repo_bindings.clone() + } + } + } + + /// Re-resolves the company's effective MCP server set: from the secret store + /// when [`HarnessDeps::secrets`] is wired (picking up console changes), else + /// the boot-resolved [`HarnessDeps::mcp_servers`] unchanged. A resolution + /// error degrades to the boot-resolved set rather than dropping MCP tools. + async fn resolve_effective_mcp( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + ) -> Vec { + match &deps.secrets { + Some(secrets) => { + let mut decls = crate::company::mcp::resolve_effective( + &company.id, + &deps.default_mcp_servers, + &company.manifest.mcp_servers, + secrets.as_ref(), + ) + .await + .unwrap_or_else(|_| deps.mcp_servers.clone()); + // Refresh any near-expiry console-OAuth credential before the + // registry is built, so an agent never sends a stale bearer. + refresh_oauth_decls(&company.id, &mut decls, secrets.as_ref()).await; + decls + } + None => deps.mcp_servers.clone(), + } + } + + /// Re-resolves the company's live overlay-agent set (issue #71) **and** its + /// operator budget overrides (issue #343): reloads the [`CompanyRecord`] + /// from [`HarnessDeps::store`] so a teammate added through the console + /// `POST .../team` route or the orchestrator's `add_agent` tool, and a cap + /// written through `PUT .../team/{id}/budget`, both reach the roster on the + /// company's next `ensure` call — the same live-re-resolution pattern as + /// [`Self::resolve_effective_mcp`]. A missing record or a store error + /// degrades to the `company` snapshot passed in (never worse than the + /// pre-#71 always-static behaviour). + /// + /// The two collections share **one** store round-trip deliberately: they + /// come off the same record, and splitting them would double the per-turn + /// read for no gain. + /// Resolves every roster member's routed workspace documents, keyed by agent + /// id (`docs/spec/runtime/orchestration/context-routing.md`). + /// + /// Runs here, in the async caller, because `build_roster` is synchronous and + /// the [`WorkspaceStore`](crate::ports::WorkspaceStore) is not — the same + /// split as the skill deltas beside it. + /// + /// **Fails soft, per agent.** A store error yields no documents for that + /// role rather than failing the rebuild: routing enriches a prompt, and a + /// company whose workspace read hiccuped should answer from a thinner prompt + /// rather than stop answering. An unwired store (`None`) resolves to an + /// empty map, which is the pre-routing behaviour exactly. + /// + /// Overlay teammates are included: they are real roster agents that + /// [`build_roster`] builds the same way, so leaving them out would give a + /// console-added teammate a silently different prompt from a manifest one. + async fn resolve_routed_context( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + overlay_agents: &[OverlayAgent], + ) -> HashMap> { + let Some(workspace) = deps.workspace.as_ref() else { + return HashMap::new(); + }; + + // A manifest agent wins an id collision, exactly as `build_roster` + // resolves one, so the overlay half skips any id already claimed. + let manifest_ids: HashSet<&str> = company + .manifest + .agents + .iter() + .map(|a| a.id.as_str()) + .collect(); + let overlay_as_manifest: Vec = overlay_agents + .iter() + .filter(|overlay| !manifest_ids.contains(overlay.id.as_str())) + .map(overlay_agent_to_manifest) + .collect(); + + let mut routed = HashMap::new(); + for agent in company.manifest.agents.iter().chain(&overlay_as_manifest) { + match crate::company::context_routing::resolve_routed_documents( + workspace.as_ref(), + &company.id, + agent, + ) + .await + { + // An agent that resolved nothing is left out of the map rather + // than stored as an empty vec: `build_roster` reads an absent id + // as "no routed documents", so the two are the same answer and + // the map stays the size of what actually routed. + Ok(documents) if documents.is_empty() => {} + Ok(documents) => { + routed.insert(agent.id.clone(), documents); + } + Err(err) => tracing::warn!( + company = %company.id, + agent = %agent.id, + error = %err, + "[context] could not read this role's routed documents; its prompt \ + goes out without them" + ), + } + } + routed + } + + async fn resolve_effective_overlay( + &self, + company: &CompanyRecord, + deps: &HarnessDeps, + ) -> EffectiveOverlay { + match deps.store.load(&company.id).await { + Ok(Some(record)) => EffectiveOverlay { + agents: record.overlay_agents, + budgets: record.overlay_budgets, + policy: record.overlay_policy, + desks: record.overlay_desks, + desk_members: record.overlay_desk_members, + desk_tools: record.overlay_desk_tools, + }, + _ => EffectiveOverlay { + agents: company.overlay_agents.clone(), + budgets: company.overlay_budgets.clone(), + policy: company.overlay_policy.clone(), + desks: company.overlay_desks.clone(), + desk_members: company.overlay_desk_members.clone(), + desk_tools: company.overlay_desk_tools.clone(), + }, + } + } + + /// The current MCP fingerprint for a company (test-only), so a freshness test + /// can assert a rebuild happened without introspecting agent internals. + #[cfg(test)] + pub async fn mcp_fingerprint_of(&self, company: &CompanyId) -> Option { + self.mcp_fingerprints.read().await.get(company).copied() + } + + /// The current overlay-agent fingerprint for a company (test-only), mirroring + /// [`Self::mcp_fingerprint_of`]. + #[cfg(test)] + pub async fn overlay_fingerprint_of(&self, company: &CompanyId) -> Option { + self.overlay_fingerprints.read().await.get(company).copied() + } + + /// The current capability-filter fingerprint for a company (test-only), so a + /// budget-freshness test can assert a rebuild happened (issue #108). + #[cfg(test)] + pub async fn capability_fingerprint_of(&self, company: &CompanyId) -> Option { + self.capability_fingerprints + .read() + .await + .get(company) + .copied() + } + + /// The current bound-repository fingerprint for a company (test-only), so a + /// bind / rotate / revoke freshness test can assert the roster was actually + /// rebuilt rather than inferring it (issue #245). + #[cfg(test)] + pub async fn repo_fingerprint_of(&self, company: &CompanyId) -> Option { + self.repo_fingerprints.read().await.get(company).copied() + } + + /// The current skill-delta fingerprint for a company (test-only), so a + /// skill-freshness test can assert a rebuild happened (issue #41). + #[cfg(test)] + pub async fn skill_fingerprint_of(&self, company: &CompanyId) -> Option { + self.skill_fingerprints.read().await.get(company).copied() + } + + /// The current budget-override fingerprint for a company (test-only), so a + /// budget-freshness test can assert the roster was actually rebuilt after a + /// console cap change rather than inferring it from the refusal (issue + /// #343). This is the observable that makes "no restart" testable. + #[cfg(test)] + pub async fn budget_fingerprint_of(&self, company: &CompanyId) -> Option { + self.budget_fingerprints.read().await.get(company).copied() + } + + /// The current billing-connection fingerprint for a company (test-only), so + /// a credential-freshness test can assert the roster was rebuilt after a key + /// was saved or rotated in Settings → Billing rather than inferring it from + /// the tool list (issues #788, #789). + #[cfg(test)] + pub async fn billing_fingerprint_of(&self, company: &CompanyId) -> Option { + self.billing_fingerprints.read().await.get(company).copied() + } + + /// The current desk-scope fingerprint for a company (test-only), so a + /// desk-scoping test can assert the roster was actually rebuilt after a + /// ceiling or seating change rather than inferring it from a refused call. + #[cfg(test)] + pub async fn desk_fingerprint_of(&self, company: &CompanyId) -> Option { + self.desk_fingerprints.read().await.get(company).copied() + } + + /// The current routed-context fingerprint for a company (test-only), so a + /// routing test can assert that editing a routed workspace note actually + /// rebuilt the roster rather than inferring it from a reply. + #[cfg(test)] + pub async fn context_fingerprint_of(&self, company: &CompanyId) -> Option { + self.context_fingerprints.read().await.get(company).copied() + } + + /// Routes a message to one agent and returns its reply, recording the turn's + /// cost. `agent_id` must name a member of the company's roster. + /// + /// Desk routing (which agent answers a group chat) is the caller's job — v1 + /// is single-responder and the WS3 chat handler picks the addressed member. + /// + /// `chat_id` is the chat/desk **thread** this turn answers (the id journaled + /// as `AgentReply.chat_id`). It rides each live turn-stream frame so the + /// console routes the in-flight tool timeline to the right thread; `None` + /// falls back to the default desk, matching the durable reply. + pub async fn run( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + deps: &HarnessDeps, + chat_id: Option<&str>, + ) -> crate::Result { + self.run_inner( + company, + agent_id, + message, + deps, + None, + LiveStream::On { chat_id }, + None, + ) + .await + } + + /// Like [`run`](Self::run) but WITHOUT live turn streaming — for a turn that + /// surfaces no operator chat bubble (a workflow agent node, which drops its + /// steps). Its transient `tool_call`/`tool_result` frames would otherwise + /// leak onto the console's live timeline and misattribute to whatever thread + /// most recently sent, so this path publishes nothing (#125 review). + pub async fn run_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + deps: &HarnessDeps, + ) -> crate::Result { + self.run_inner( + company, + agent_id, + message, + deps, + None, + LiveStream::Off, + None, + ) + .await + } + + /// Routes a message to one agent with an operator **steer** control installed + /// (issue #111), so a dispatched task / desk delegation can be paused, + /// cancelled, or redirected mid-flight. Otherwise identical to + /// [`run`](Self::run) — same retrieve→inject, cost accounting, and + /// memory-writeback. The steer hook fires only between tool-loop iterations. + /// `chat_id` routes the live turn-stream frames exactly as in [`run`](Self::run). + /// + /// `run_sink` is the dispatched attempt this turn belongs to, when it + /// belongs to one (issue #242) — a desk turn a *dispatched card* handed its + /// work to records into the card's run, while the same delegation reached + /// from operator chat passes `None`. + #[allow(clippy::too_many_arguments)] + pub async fn run_steered( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + deps: &HarnessDeps, + control: &SteerControl, + chat_id: Option<&str>, + run_sink: Option>, + ) -> crate::Result { + self.run_inner( + company, + agent_id, + message, + deps, + Some(control), + LiveStream::On { chat_id }, + run_sink, + ) + .await + } + + /// Like [`run_steered`](Self::run_steered) but WITHOUT live turn streaming — + /// for a dispatched task card, which discards its steps and shows no chat + /// bubble. Its transient turn frames must not reach the live console + /// timeline (they'd misattribute to a chat thread), so this path publishes + /// nothing while still honouring the operator steer control (#125 review). + pub async fn run_steered_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + deps: &HarnessDeps, + control: &SteerControl, + run_sink: Option>, + ) -> crate::Result { + self.run_inner( + company, + agent_id, + message, + deps, + Some(control), + LiveStream::Off, + run_sink, + ) + .await + } + + /// The plan-level total-token ceiling, as a refusal or nothing. + /// + /// Extracted from [`run_inner`](Self::run_inner) so the confined turn + /// (issue #416) is gated by the *same* ceiling rather than a second copy of + /// the rule: a turn that reaches nothing still spends model tokens, so a + /// tenant past its cap must not be able to keep spending through the + /// copilot. + async fn total_ceiling_refusal( + company: &CompanyId, + agent_id: &str, + deps: &HarnessDeps, + ) -> Option { + let plan = deps.plan.as_ref()?; + plan.total_budget?; + match deps.meter.as_deref() { + Some(meter) => { + let since = plan.period.period_start_millis(crate::ports::now_millis()); + match meter.query(company, since).await { + Ok(samples) => { + let spent = capability_budget::tokens_in(&samples); + if plan.total_exhausted(spent) { + tracing::info!( + company = %company, + agent = agent_id, + spent, + "[capability-budget] total token ceiling reached; refusing dispatch (no model call) until the period resets" + ); + return Some(TurnOutcome { + reply: TOTAL_BUDGET_EXHAUSTED_NOTICE.to_string(), + steps: Vec::new(), + // No model call ran, so no cap was reached + // (issue #926). A refusal is not a pause. + hit_iteration_cap: false, + // And no in-turn hook fired, because no turn + // ran (issue #1032). The reply already IS the + // budget notice; labelling this as a halt too + // would tell the operator the same thing twice. + halted_for_spend: None, + }); + } + } + Err(error) => { + tracing::warn!( + company = %company, + %error, + "[capability-budget] total-ceiling spend query failed; not hard-refusing — deferring to the per-namespace fail-closed roster" + ); + } + } + } + None => { + tracing::warn!( + company = %company, + "[capability-budget] no usage meter; cannot enforce the total token ceiling — deferring to the per-namespace fail-closed roster" + ); + } + } + None + } + + /// Runs one **confined** turn (issue #416): an ephemeral agent with no + /// tools, no company memory and no roster identity, for a question about one + /// object rather than about the company. + /// + /// Deliberately not a variant of [`run_inner`](Self::run_inner), because the + /// two differ in what they are allowed to touch rather than in a flag: + /// + /// * the agent is **built here and dropped after**, so it is never in the + /// pooled roster and cannot be addressed, dispatched or delegated to; + /// * there is **no retrieve→inject** — the company's prior task outcomes are + /// not prepended to the message, so the model cannot answer from work it + /// was not asked about; + /// * there is **no memory writeback** — the exchange leaves nothing for a + /// later company turn to retrieve, so a confined conversation cannot + /// become unconfined context tomorrow. + /// + /// What it does share: the plan-level token ceiling (spend is spend), live + /// turn streaming onto the addressed thread, and cost recording, so a + /// confined turn is billed and observable exactly like any other. + pub async fn run_confined( + &self, + company: &CompanyId, + company_name: &str, + message: &str, + deps: &HarnessDeps, + chat_id: Option<&str>, + confinement: &confine::Confinement, + ) -> crate::Result { + if let Some(refusal) = + Self::total_ceiling_refusal(company, confine::CONFINED_AGENT_ID, deps).await + { + return Ok(refusal); + } + + let agent = CompanyAgent { + agent_id: confine::CONFINED_AGENT_ID.to_string(), + role: "Workflow copilot".to_string(), + // A confined turn carries no manifest teammate, so there is no + // per-agent daily cap to read; the company-wide ceiling above is the + // one that applies to it. + budget_usd_daily: None, + agent: Mutex::new(confine::build_confined_agent( + company, + company_name, + confinement, + deps, + )?), + }; + + let stream_ctx = Some(crate::turn_stream::TurnStreamCtx { + company: company.clone(), + agent_id: confine::CONFINED_AGENT_ID.to_string(), + chat_id: chat_id + .map(str::to_string) + .unwrap_or_else(|| crate::server::ops::language::DEFAULT_DESK.to_string()), + }); + + // The message goes to the model AS SENT. This is the retrieve→inject + // step's absence, and it is the difference between "grounded in one + // workflow" and "confined to one workflow". + let (outcome, turn_costs) = agent + .run_with_steer(message, None, stream_ctx, None) + .await?; + + let provider_slug = deps.provider.telemetry_provider_id(); + for turn_cost in &turn_costs { + record_turn_cost( + turn_cost, + confine::CONFINED_AGENT_ID, + &provider_slug, + company, + deps.store.as_ref(), + deps.meter.as_deref(), + None, + ) + .await?; + } + + Ok(outcome) + } + + #[allow(clippy::too_many_arguments)] + async fn run_inner( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + deps: &HarnessDeps, + steer: Option<&SteerControl>, + live: LiveStream<'_>, + run_sink: Option>, + ) -> crate::Result { + let agent = { + let guard = self.agents.read().await; + let roster = guard + .get(company) + .ok_or_else(|| OpenCompanyError::CompanyNotFound(company.to_string()))?; + roster + .iter() + .find(|a| a.agent_id == agent_id) + .cloned() + .ok_or_else(|| { + OpenCompanyError::InvalidRequest(format!( + "agent '{agent_id}' is not on company '{company}' roster" + )) + })? + }; + + // Renew the agent's sandbox directory at the moment it acts (issue + // #409). `build_agent` already created it, but a roster is built once + // and then cached behind fingerprints — and handed *across* an in-place + // rebuild — so a workspace that goes missing afterwards (a restored or + // wiped data dir, an operator clearing the tree, a boot that raced a + // not-yet-mounted volume) would otherwise stay missing for the life of + // the process, and every relative file write would be refused as if it + // had tried to escape the sandbox. Two syscalls on the already-exists + // path, against a turn that is about to call a model — not worth + // deferring off the runtime thread. + // + // Deliberately not fatal, for the same reason `build_agent`'s attempt is + // not: an agent with no file grant runs a perfectly good turn without + // this directory. The `error!` (not `warn!`) records the one condition + // under which the misdirecting guard message can still be reached, so it + // is greppable next to the refusal it explains. Both of those are the + // right calls and issue #449 does not change either. + // + // What #449 changes is only how often it is *said*. A workspace root + // that cannot be written — a volume that failed to mount, a path that + // resolves onto a file — fails identically on every dispatch, so the + // unconditional `error!` emitted one byte-identical line per turn, + // forever, with nothing distinguishing the thousandth from the first. + // The state is edge-triggered instead: the first failure reads exactly + // as it did before, the repeats are silent, and a recovery gets one + // `info!` so a reader who saw the error learns when it ended. The + // attempt itself still runs every dispatch — see + // `note_workspace_attempt` for why memoising it would be a regression. + let attempt = build::ensure_agent_workspace(&deps.workspace_root, company, agent_id); + let report = self.note_workspace_attempt(company, agent_id, attempt.is_err()); + if !report.is_silent() { + let workspace = build::agent_workspace(&deps.workspace_root, company, agent_id); + match attempt { + Err(error) => tracing::error!( + company = %company, + agent = agent_id, + workspace = %workspace.display(), + %error, + "[harness] could not create the agent workspace before dispatch; relative file writes will be refused (the refusal will read as a workspace escape, but the cause is this missing directory)" + ), + Ok(_) => tracing::info!( + company = %company, + agent = agent_id, + workspace = %workspace.display(), + "[harness] agent workspace is available again; the earlier creation failure has cleared and relative file writes work" + ), + } + } + + // Plan-level total-token ceiling (issue #188): a HARD dispatch refusal + // that never reaches the model once the tenant's total period spend + // crosses the cap. The per-namespace budget gate in `ensure` is *soft* — + // it only trims which exec tools the roster carries; an exhausted + // tenant's turn still runs on intrinsic tools and burns model tokens. + // This closes that gap by refusing dispatch outright, before any model + // call, on every path that funnels through `run_inner` (operator chat, + // task, steered/background). We return early here — before retrieve→ + // inject and the memory writeback — so a refused turn costs nothing and + // leaves no fabricated outcome in the memory store. + // + // Fail-closed tradeoff (issue #188): the hard refusal fires ONLY when + // spend is actually readable. With no meter, or a meter whose query + // errors, we do NOT brick the tenant on a transient read failure — we + // fall through to run the turn, which the per-namespace fail-closed path + // in `resolve_filter`/`ensure` has already stripped of every exec tool. + // A `warn!` records the deferral. Refusing every turn on a flaky meter + // read would be a strictly worse failure mode than letting an + // intrinsic-tools-only turn through. + if let Some(refusal) = Self::total_ceiling_refusal(company, agent_id, deps).await { + return Ok(refusal); + } + + // Per-agent daily spend cap (issue #304): the same HARD, pre-model-call + // refusal as the ceiling above, scoped to ONE teammate. + // + // This is the layer that matters most in practice. The manifest's + // `budget_usd_daily` was validated, persisted and passed to + // `ApprovalPolicy` — where it sat on a field with no reader. But the + // dominant spend stream is not tool calls at all, it is inference, and + // inference never reaches a `ToolPolicy`. Gating only priced tool calls + // (the policy arm) would leave a capped teammate free to burn its budget + // many times over on model turns alone, which is how the cap came to be + // decorative in the first place. + // + // Refused BEFORE retrieve→inject and the memory writeback, exactly like + // the total ceiling, so a refused turn costs nothing and leaves no + // fabricated outcome in the store. The reply names the teammate, the cap + // and the reset — never a bare failure. + // + // FAIL-OPEN, mirroring #188's documented tradeoff: with no meter, or a + // meter whose query errors, we warn and run the turn. Bricking a + // company's cognition on a flaky read would be a strictly worse failure + // mode than one day of overspend, and there is no operator recourse at + // turn level (unlike the policy arm, whose park a human can approve — + // which is why THAT layer fails closed and this one does not). + if let Some(cap) = agent.budget_usd_daily { + match deps.meter.as_deref() { + Some(meter) => { + let since = crate::metering::utc_day_start_millis(crate::ports::now_millis()); + match meter.query(company, since).await { + Ok(samples) => { + let spent = crate::metering::usd_spent_by_agent(&samples, agent_id); + if spent >= cap { + tracing::info!( + company = %company, + agent = agent_id, + spent, + cap, + "[agent-budget] daily spend cap reached; refusing dispatch (no model call) until 00:00 UTC" + ); + return Ok(TurnOutcome { + reply: agent_budget_exhausted_notice(agent_id, cap), + steps: Vec::new(), + // No model call ran, so no cap was reached + // (issue #926). A refusal is not a pause. + hit_iteration_cap: false, + // Same teammate cap, refused BEFORE the + // turn (issue #1032). The in-turn brake + // never armed, and the reply above already + // names the cap it refused against. + halted_for_spend: None, + }); + } + } + Err(error) => { + tracing::warn!( + company = %company, + agent = agent_id, + %error, + "[agent-budget] daily-spend query failed; running the turn rather than bricking this teammate" + ); + } + } + } + None => { + tracing::warn!( + company = %company, + agent = agent_id, + "[agent-budget] no usage meter; the per-agent daily spend cap cannot be enforced on this host" + ); + } + } + } + + // Retrieve→inject: pull the top-K prior task outcomes relevant to this + // message and prepend them as context. On a cold store this yields no + // hits and the message is passed through unchanged. + let hits = deps + .context + .search(company, message, memory_loop::RETRIEVE_TOP_K) + .await?; + let augmented = memory_loop::inject(message, &hits); + + // Run the turn and record its real cost. `CompanyAgent::run` reads each + // attempt's token/cost totals from openhuman's public `last_turn_usage()` + // accessor and returns one entry per attempt (two when the empty-response + // wrapper retried once). A zero-usage attempt (offline provider) writes + // nothing, so the inert-metering contract holds. + // Live tool-call streaming: for a turn that surfaces an operator chat + // bubble (`live`), hand the runner the routing context so it tees each + // progress event onto the company's transient turn-stream bus as it + // happens (the console renders the timeline live). Background turns — + // dispatched task cards and workflow agent nodes — pass `live = false` + // and stream nothing, since they carry no chat thread to render onto and + // their frames would otherwise misattribute to the active chat (#125 + // review). Either way the durable `TurnStep`s still fold from the same + // buffered events at turn end. + let stream_ctx = match live { + LiveStream::On { chat_id } => Some(crate::turn_stream::TurnStreamCtx { + company: company.clone(), + agent_id: agent_id.to_string(), + // The chat/desk thread this turn answers — the same id journaled + // as `AgentReply.chat_id`, so the console keys the live timeline + // on it and concurrent turns on different threads never + // cross-attribute. Falls back to the default desk to match the + // durable reply when the caller addressed no desk (e.g. an API + // client that omits `chat`). + chat_id: chat_id + .map(str::to_string) + .unwrap_or_else(|| crate::server::ops::language::DEFAULT_DESK.to_string()), + }), + LiveStream::Off => None, + }; + let (outcome, turn_costs) = agent + .run_with_steer(&augmented, steer, stream_ctx, run_sink.clone()) + .await?; + // Issue #242: fold this turn's spend into the attempt it belongs to. + // Per turn, not once at the end, so a redirect re-run and a delegate's + // turn both count — an attempt's cost is what the attempt spent. This is + // a second *reader* of `turn_costs`, not a second writer: the ledger and + // the usage meter below stay the only places money is recorded. + if let Some(sink) = run_sink.as_ref() { + for turn_cost in &turn_costs { + sink.add_usage(turn_cost); + } + } + // Attribute cost to the provider this turn actually resolved to. With a + // per-tenant [`TenantProvider`](crate::harness::provider::TenantProvider) + // a console BYOK switch changes the slug between turns, so read it live + // rather than trusting the static `deps.provider_slug` baked at build. + let provider_slug = deps.provider.telemetry_provider_id(); + for turn_cost in &turn_costs { + record_turn_cost( + turn_cost, + agent_id, + &provider_slug, + company, + deps.store.as_ref(), + deps.meter.as_deref(), + // Issue #242: attribute the sample to the attempt this turn ran + // under, so "what did this run cost?" is answerable from the + // meter as well as from the run row. + run_sink.as_ref().map(|s| s.run_id()), + ) + .await?; + } + + // Store: persist the outcome (original task + reply) so it compounds + // into later turns. Without this the harness never writes memory back. + // SECURITY: the reply **text only** — the scrubbed `outcome.steps` never + // enter the memory store, so a step detail can never be retrieved and + // re-injected into a later turn. + if !matches!( + steer.and_then(SteerControl::pending), + Some(SteerAction::Cancel) + ) { + deps.context + .put( + company, + memory_loop::outcome_chunk(agent_id, message, &outcome.reply), + ) + .await?; + } + + Ok(outcome) + } + + /// Number of companies currently resident in the pool (test/observability). + pub async fn resident_companies(&self) -> usize { + self.agents.read().await.len() + } + + /// The agent ids this pool currently holds for `company`, in roster order. + /// + /// Observability for the per-harness split: a pool serving one named harness + /// should hold only that harness's agents, and this is how that is checked + /// without reaching into the lock. + pub async fn agent_ids(&self, company: &CompanyId) -> Vec { + self.agents + .read() + .await + .get(company) + .map(|agents| agents.iter().map(|a| a.agent_id.clone()).collect()) + .unwrap_or_default() + } +} + +/// A stable fingerprint of an effective MCP server set, used to detect a console +/// change (add / remove / enable-toggle / token rotation) between +/// [`HarnessPool::ensure`] calls. Hashes only non-secret configuration plus the +/// credential substrings — the resulting `u64` is non-reversible and never +/// surfaces anywhere, so it is not a credential leak, and hashing the credential +/// substrings means a rotate-token also invalidates the cached roster. +fn mcp_fingerprint(decls: &[McpServerDecl]) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + decls.len().hash(&mut hasher); + for decl in decls { + decl.name.hash(&mut hasher); + decl.endpoint.hash(&mut hasher); + decl.enabled.hash(&mut hasher); + decl.description.hash(&mut hasher); + decl.allowed_tools.hash(&mut hasher); + decl.disallowed_tools.hash(&mut hasher); + decl.timeout_secs.hash(&mut hasher); + auth_kind(&decl.auth).hash(&mut hasher); + for secret in decl.auth.secret_values() { + secret.hash(&mut hasher); + } + } + hasher.finish() +} + +/// A small discriminant for an [`AuthMaterial`] variant, for the fingerprint. +fn auth_kind(material: &crate::company::mcp::AuthMaterial) -> u8 { + use crate::company::mcp::AuthMaterial::*; + match material { + None => 0, + Bearer(_) => 1, + Header { .. } => 2, + QueryParam { .. } => 3, + OAuth { .. } => 4, + } +} + +/// Refreshes any near-expiry console-OAuth credential in `decls` before the +/// registry is built, re-persisting the rotated token **write-only** so agents +/// never send an expired bearer. Per-tenant analogue of OpenHuman's +/// `mcp_registry::oauth::refresh_if_expired`. A refresh failure is non-fatal — +/// the old token is kept and the next `401` re-prompts sign-in. +#[cfg(feature = "mcp")] +async fn refresh_oauth_decls( + company: &CompanyId, + decls: &mut [McpServerDecl], + secrets: &dyn SecretStore, +) { + use crate::company::mcp_oauth; + + for decl in decls.iter_mut() { + if !mcp_oauth::needs_refresh(&decl.auth, 60) { + continue; + } + let Some(new_material) = mcp_oauth::refresh(&decl.auth).await else { + continue; + }; + match crate::company::mcp::store_auth(company, &decl.name, &new_material, secrets).await { + Ok(()) => decl.auth = new_material, + Err(err) => log::warn!( + "[mcp-oauth] failed to persist refreshed token for `{}`: {}", + decl.name, + err.code() + ), + } + } +} + +/// Without the `mcp` feature there is no OAuth credential to refresh, so this is +/// a no-op (keeps `resolve_effective_mcp` uniform across the two builds). +#[cfg(not(feature = "mcp"))] +async fn refresh_oauth_decls( + _company: &CompanyId, + _decls: &mut [McpServerDecl], + _secrets: &dyn SecretStore, +) { +} + +/// A stable fingerprint of an overlay-agent set (issue #71), used to detect a +/// teammate add/remove/edit between [`HarnessPool::ensure`] calls. Mirrors +/// [`mcp_fingerprint`]'s shape; no secrets are involved here so there is +/// nothing to scrub — an [`OverlayAgent`] is display data. +fn overlay_fingerprint(agents: &[OverlayAgent]) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + agents.len().hash(&mut hasher); + for agent in agents { + agent.id.hash(&mut hasher); + agent.name.hash(&mut hasher); + agent.role.hash(&mut hasher); + agent.description.hash(&mut hasher); + // Issue #661 / L5: a grant edit changes the roster the harness must + // build, so it has to move this fingerprint — otherwise a re-grant would + // persist and be silently ignored until the next process restart, the + // same staleness the tier/skill fingerprints exist to prevent. Hashed in + // order (an operator's own list), length folded in first via the slice + // length above so `["a","b"]` cannot collide with `["ab"]`. + agent.tools.hash(&mut hasher); + } + hasher.finish() +} + +/// A stable hash of the operator's `[policy]` override, so a console tier change +/// rebuilds the roster on the company's next `ensure` (issue #562). +/// +/// # Why this axis has to exist at all +/// +/// `ApprovalPolicy` is constructed in [`build_roster`], **once per roster +/// build** — not once per call. The roster is cached and rebuilt only when one +/// of the fingerprints in the staleness check moves. So without this function a +/// console tier change would be written, persisted, and then **silently ignored +/// until the process restarted**: the write route would return `204`, the +/// console would show the new tier, and every agent would keep running the old +/// one. That is the same failure the skill-delta fingerprint above exists to +/// prevent, and it is invisible from the outside. +/// +/// # What is hashed, and what deliberately is not +/// +/// - `mode` and `always_approve` are hashed — they are what the gate reads. +/// - `always_approve` is hashed **in order**, unlike the budget set. The order +/// is the operator's own list as they wrote it, not an accumulation of +/// independent rows, so a reorder is a real edit rather than a spurious +/// difference. Its length is folded in first so `["a","b"]` cannot collide +/// with `["ab"]`. +/// - The `Some`/`None` distinction is hashed for both fields, because "not +/// overridden" and "overridden to the manifest's current value" must stay +/// apart: the manifest can change under a rebuild, and collapsing them would +/// pin the override to a value the operator never chose. +/// - **Attribution (`set_by`, `at_millis`) is deliberately NOT hashed**, for the +/// same reason the budget fingerprint omits it: who set the tier and when +/// changes nothing an agent can act on, and folding it in would rebuild the +/// roster — dropping live agent sessions — on a save that re-set the same tier. +fn policy_fingerprint(override_: Option<&PolicyOverride>) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + match override_ { + None => 0u8.hash(&mut hasher), + Some(entry) => { + 1u8.hash(&mut hasher); + match &entry.mode { + Some(mode) => { + 1u8.hash(&mut hasher); + mode.hash(&mut hasher); + } + None => 0u8.hash(&mut hasher), + } + match &entry.always_approve { + Some(kinds) => { + 1u8.hash(&mut hasher); + kinds.len().hash(&mut hasher); + for kind in kinds { + kind.hash(&mut hasher); + } + } + None => 0u8.hash(&mut hasher), + } + } + } + hasher.finish() +} + +/// The live overlay state one roster rebuild is resolved against. +/// +/// A struct rather than the tuple this used to be: it grew past the point where +/// positional returns stay readable, and — more to the point — the desk fields +/// were added because desks now decide *capability*, so a caller silently +/// binding `desk_tools` to the `desks` position would hand every teammate the +/// wrong tool belt with nothing to catch it. +pub(crate) struct EffectiveOverlay { + pub agents: Vec, + pub budgets: Vec, + pub policy: Option, + pub desks: Vec, + pub desk_members: Vec, + pub desk_tools: std::collections::BTreeMap>, +} + +/// Fingerprints the routed workspace documents a roster's personas are built +/// from — **over their bodies**, not their names. +/// +/// Hashing the content is the whole point. The routing table is manifest data +/// and does not move when an operator edits a note, so a name-only hash would +/// leave the edit invisible: the persona is assembled once per roster, and the +/// fast path would keep serving a prompt quoting the old text until the process +/// restarted. That is precisely the staleness the routing layer exists to avoid. +/// +/// Sorted by agent id before hashing, for the reason [`budget_fingerprint`] +/// documents — a `HashMap` has no order, and an order-sensitive hash would drop +/// every live agent session on a rebuild that changed nothing. +fn routed_context_fingerprint(routed: &HashMap>) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut ordered: Vec<(&String, &Vec<(String, String)>)> = routed.iter().collect(); + ordered.sort_by(|a, b| a.0.cmp(b.0)); + + let mut hasher = DefaultHasher::new(); + ordered.len().hash(&mut hasher); + for (agent_id, documents) in ordered { + agent_id.hash(&mut hasher); + documents.len().hash(&mut hasher); + for (path, body) in documents { + path.hash(&mut hasher); + body.hash(&mut hasher); + } + } + hasher.finish() +} + +/// Fingerprints the desk scoping a roster's grants are resolved through: which +/// desks exist, who sits on them, and what each one's tool ceiling is. +/// +/// All three axes are hashed together because all three feed one answer — an +/// agent's effective grant. Seating a teammate on a restricted desk narrows its +/// belt just as surely as editing that desk's ceiling does, so a fingerprint +/// over the ceilings alone would leave a membership change invisible until the +/// next restart, which is the staleness bug this whole fingerprint set exists to +/// prevent. +/// +/// Sorted before hashing, for the reason [`budget_fingerprint`] documents: the +/// write routes push and retain rather than maintain an order, and an +/// order-sensitive hash would drop every live agent session on a save that +/// changed nothing an agent can observe. (`desk_tools` is a `BTreeMap` and so is +/// already ordered by construction.) +fn desk_scope_fingerprint( + desks: &[OverlayDesk], + members: &[OverlayDeskMember], + tools: &std::collections::BTreeMap>, +) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + + let mut desk_ids: Vec<&str> = desks.iter().map(|desk| desk.id.as_str()).collect(); + desk_ids.sort_unstable(); + desk_ids.hash(&mut hasher); + + let mut seats: Vec<(&str, &str)> = members + .iter() + .map(|seat| (seat.desk_id.as_str(), seat.agent_id.as_str())) + .collect(); + seats.sort_unstable(); + seats.hash(&mut hasher); + + tools.len().hash(&mut hasher); + for (desk_id, ceiling) in tools { + desk_id.hash(&mut hasher); + ceiling.hash(&mut hasher); + } + + hasher.finish() +} + +/// A stable fingerprint of a company's operator budget-override set (issue +/// #343), used to detect a cap set / changed / cleared / reset between +/// [`HarnessPool::ensure`] calls. Mirrors [`overlay_fingerprint`]'s shape; a +/// [`BudgetOverride`] holds no secret. +/// +/// Two details carry weight: +/// +/// - The set is **sorted by `agent_id`** first, because the write routes push +/// and retain rather than maintain an order, and an order-sensitive hash would +/// rebuild the roster (dropping live agent sessions) on a save that changed +/// nothing an agent can observe. +/// - The cap is hashed as an `Option` **discriminant plus `f64::to_bits`**, not +/// through `PartialEq`. `f64` is not `Hash`, and going through bits is also +/// what keeps `Some(0.0)` distinct from `None` in the hash — the very +/// distinction the issue insists must not collapse. `to_bits` additionally +/// makes the hash total over values `PartialEq` would call incomparable. +fn budget_fingerprint(overrides: &[BudgetOverride]) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut ordered: Vec<&BudgetOverride> = overrides.iter().collect(); + ordered.sort_by(|a, b| a.agent_id.cmp(&b.agent_id)); + + let mut hasher = DefaultHasher::new(); + ordered.len().hash(&mut hasher); + for entry in ordered { + entry.agent_id.hash(&mut hasher); + match entry.budget_usd_daily { + Some(cap) => { + 1u8.hash(&mut hasher); + cap.to_bits().hash(&mut hasher); + } + None => 0u8.hash(&mut hasher), + } + } + // Attribution is deliberately NOT hashed: who set the cap and when changes + // nothing an agent can act on, and folding it in would rebuild the roster + // (discarding live sessions) every time the same value was re-saved. + hasher.finish() +} + +/// A stable fingerprint of a company's operator skill-delta set (issue #41), +/// used to detect a skill authored / edited / enabled / disabled between +/// [`HarnessPool::ensure`] calls. Mirrors [`mcp_fingerprint`]'s shape. +/// +/// The deltas are **sorted by `slug`** before hashing because +/// [`SkillStateStore::list`](crate::ports::skills_state::SkillStateStore::list) +/// gives no ordering contract — an order-sensitive hash would thrash the roster +/// (and drop live agent conversation state) whenever the store returned the +/// same skills in a different row order. The full `custom_doc` body is hashed so +/// an *edited* skill (same slug, new content) also triggers a rebuild. No +/// The disabling [`SkillState`] deltas a company's `[globals].disable` implies. +/// +/// One per `skill:` entry, and nothing else: an entry naming another kind +/// is that kind's business, and manifest validation has already refused an entry +/// naming nothing at all. +pub(crate) fn globals_skill_disables(disable: &[String]) -> Vec { + disable + .iter() + .filter_map(|entry| entry.strip_prefix("skill:")) + .map(|slug| SkillState { + slug: slug.to_string(), + enabled: false, + // The shared library is where these skills are authored, so that is + // what they are a delta over. The value is inert here in any case: + // this delta is synthesized per rebuild, never stored, and only its + // `enabled = false` is read. + source: crate::ports::SkillSource::Registry, + custom_doc: None, + }) + .collect() +} + +/// secrets are involved — a skill delta is operator-authored content. +fn skill_delta_fingerprint(deltas: &[SkillState]) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut ordered: Vec<&SkillState> = deltas.iter().collect(); + ordered.sort_by(|a, b| a.slug.cmp(&b.slug)); + + let mut hasher = DefaultHasher::new(); + ordered.len().hash(&mut hasher); + for delta in ordered { + delta.slug.hash(&mut hasher); + delta.enabled.hash(&mut hasher); + delta.source.hash(&mut hasher); + delta.custom_doc.hash(&mut hasher); + } + hasher.finish() +} + +/// Fingerprint of a company's bound repositories (issue #245). +/// +/// Over `(key, token_fingerprint, branches)`, sorted by key, because those are +/// exactly the three things a rebuild has to notice: +/// +/// * **key** — a bind adds one, a revoke removes one, and either changes what +/// `repo_checkout` can resolve and what its description names; +/// * **token fingerprint** — a rotation leaves the key alone, and a *revoked* +/// credential blanks it while the key survives, so keying on the set of +/// repositories would leave an agent holding a tool over a binding that can no +/// longer fetch; +/// * **branches** — the set a checkout may name, and the only other field the +/// tools read. +/// +/// Deliberately not `size_bytes` or `last_fetched_millis`: both move on every +/// fetch, and a fetch is something the agent's own tool does — folding them in +/// would rebuild the roster after every checkout, for no change an agent can +/// observe. +fn repo_binding_fingerprint(bindings: &[crate::runtime::repo_manager::types::RepoBinding]) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut ordered: Vec<&crate::runtime::repo_manager::types::RepoBinding> = + bindings.iter().collect(); + ordered.sort_by(|a, b| a.key.cmp(&b.key)); + + let mut hasher = DefaultHasher::new(); + ordered.len().hash(&mut hasher); + for binding in ordered { + binding.key.hash(&mut hasher); + binding.token_fingerprint.hash(&mut hasher); + binding.branches.hash(&mut hasher); + } + hasher.finish() +} + +/// Build every roster agent for a company: every manifest `[[agent]]`, plus +/// every operator- or orchestrator-added [`OverlayAgent`] (issue #71 — Active +/// Runtime Teammates) that does not collide with a manifest agent id. +/// +/// Overlay teammates were presentation-only before this cell (listed in the +/// console Team tab but never addressable); this promotes each one into a real +/// [`CompanyAgent`] with the same shape [`build::build_agent`] gives a manifest +/// agent — a standard (company-wide) tool grant, no cognition tier (the +/// default `chat-v1` model), and never the orchestrator. A manifest agent +/// always wins an id collision: the version-controlled roster is authoritative, +/// and [`orchestrator::orchestrator_id`] only ever looks at `manifest.agents`, +/// so an overlay teammate can never become the orchestrator. +/// +/// `skill_deltas` are the company's operator skill overrides (fetched once by +/// the async caller); every agent folds them into its effective skill set. +/// +/// `routed_context` maps an agent id to the workspace documents routed into its +/// system prompt, resolved by the async caller for the same reason +/// `skill_deltas` is — this function is synchronous and the `WorkspaceStore` is +/// not. An agent absent from the map gets no routed documents, which is the +/// correct reading for a company with no workspace store wired: fail closed to +/// the pre-routing prompt rather than to a half-populated one. +/// Whether `deps` builds `agent_id`. +/// +/// `serves: None` is the whole roster — every pre-harness caller, and the +/// single-harness case that is still the overwhelming majority. +fn serves(deps: &HarnessDeps, agent_id: &str) -> bool { + match deps.serves.as_ref() { + None => true, + Some(ids) => ids.contains(agent_id), + } +} + +pub(crate) fn build_roster( + company: &CompanyRecord, + deps: &HarnessDeps, + skill_deltas: &[SkillState], + routed_context: &HashMap>, +) -> crate::Result>> { + // Issue #562: the policy in force, not the one the manifest shipped with — + // the same relationship `effective_budget` (issue #343) has to the manifest + // cap, and resolved through the same record so the console and this gate + // cannot disagree about which tier is live. + // + // Owned rather than borrowed because the effective value is a field-wise + // merge of the override and the manifest, so there may be nothing to borrow. + let effective = company.effective_policy(); + let policy: &Policy = &effective; + let company_name = &company.manifest.company.name; + let allow = &company.manifest.tools.allow; + // The orchestrator agent (tier `orchestrator`, else the first agent) receives + // the delegating-orchestrator persona + tools (issue #53). + let orchestrator = orchestrator::orchestrator_id(&company.manifest.agents); + + let mut roster = + Vec::with_capacity(company.manifest.agents.len() + company.overlay_agents.len()); + + for manifest_agent in &company.manifest.agents { + // When these deps serve one named harness, build only the agents bound + // to it — every other agent is another pool's, holding another + // provider. Skipping here rather than filtering afterwards is what keeps + // the unbuilt agents from ever standing up a model client. + if !serves(deps, &manifest_agent.id) { + continue; + } + // Issue #343: the cap in force, not the one the manifest shipped with. + // `effective_budget` is an operator override when one is stored and the + // manifest value otherwise, so a console cap change reaches BOTH readers + // built below — the `ApprovalPolicy` arm and `CompanyAgent`'s copy that + // the L1 dispatch gate reads — from this one call. + let effective_budget = company.effective_budget(&manifest_agent.id); + let mut agent_policy = ApprovalPolicy::new(policy, effective_budget) + .with_requests(deps.approval_requests.clone()) + // Issue #243: stamp who the parked effect belongs to, so approving it + // can hand the grant back to this agent rather than to nobody. + .with_agent(manifest_agent.id.clone()); + // Issue #304: give the policy something to measure `budget_usd_daily` + // against. Only wired when the host has a meter — without one the cap + // arm stays inert and warns once, rather than parking every priced call + // on a host that can never answer the question. + if let Some(meter) = deps.meter.as_ref() { + agent_policy = agent_policy.with_spend(meter.clone(), company.id.clone()); + } + let is_orchestrator = orchestrator.as_deref() == Some(manifest_agent.id.as_str()); + // Three-level narrowing: company → the desks this teammate sits on → + // the teammate itself. `agent_desk_tools` resolves through the record's + // *effective* desk membership, so a console-seated member is scoped by + // its desk exactly as a manifest one is. + let desk_tools = company.agent_desk_tools(&manifest_agent.id); + let desk_allows: Vec<&[String]> = desk_tools.iter().map(Vec::as_slice).collect(); + let grants = agent_scoped_grants(allow, &desk_allows, &manifest_agent.tools); + let agent = build::build_agent( + &company.id, + company_name, + manifest_agent, + agent_policy, + deps, + &grants, + skill_deltas, + routed_context + .get(&manifest_agent.id) + .map(Vec::as_slice) + .unwrap_or(&[]), + is_orchestrator, + )?; + roster.push(Arc::new(CompanyAgent { + agent_id: manifest_agent.id.clone(), + role: manifest_agent.role.clone(), + budget_usd_daily: effective_budget, + agent: Mutex::new(agent), + })); + } + + // Issue #71 — Active Runtime Teammates (minimal slice): promote every + // operator/orchestrator-added overlay teammate into a real roster agent + // too, skipping any id already claimed by a manifest agent. + let manifest_ids: HashSet<&str> = company + .manifest + .agents + .iter() + .map(|a| a.id.as_str()) + .collect(); + for overlay in &company.overlay_agents { + if manifest_ids.contains(overlay.id.as_str()) { + continue; + } + if !serves(deps, &overlay.id) { + continue; + } + let manifest_agent = overlay_agent_to_manifest(overlay); + // Issue #343: an overlay teammate has no manifest row to carry a cap, so + // before the override existed it was unconditionally uncapped — the "v1 + // limitation" this lifts. `effective_budget` gives it a stored cap when + // an operator set one, and `None` (as before) when nobody has. + let effective_budget = company.effective_budget(&manifest_agent.id); + let mut agent_policy = ApprovalPolicy::new(policy, effective_budget) + .with_requests(deps.approval_requests.clone()) + // An overlay teammate is a real roster agent and re-dispatches the + // same way a manifest one does (issue #243). + .with_agent(manifest_agent.id.clone()); + if let Some(meter) = deps.meter.as_ref() { + agent_policy = agent_policy.with_spend(meter.clone(), company.id.clone()); + } + // An overlay teammate is scoped by its desks the same as a manifest one: + // it can be seated on a desk, and a desk ceiling that applied to only + // half its members would not be a ceiling. + let desk_tools = company.agent_desk_tools(&manifest_agent.id); + let desk_allows: Vec<&[String]> = desk_tools.iter().map(Vec::as_slice).collect(); + let grants = agent_scoped_grants(allow, &desk_allows, &manifest_agent.tools); + let agent = build::build_agent( + &company.id, + company_name, + &manifest_agent, + agent_policy, + deps, + &grants, + skill_deltas, + routed_context + .get(&manifest_agent.id) + .map(Vec::as_slice) + .unwrap_or(&[]), + /* is_orchestrator */ false, + )?; + roster.push(Arc::new(CompanyAgent { + agent_id: manifest_agent.id.clone(), + role: manifest_agent.role.clone(), + budget_usd_daily: effective_budget, + agent: Mutex::new(agent), + })); + } + + Ok(roster) +} + +/// Converts an operator-added [`OverlayAgent`] into the manifest agent shape +/// [`build::build_agent`] consumes: an empty `tools` list (so +/// [`agent_effective_grants`] falls back to the full company `[tools].allow` +/// — the "standard tool grant"), no cognition tier (→ the default `chat-v1` +/// model), and no manifest budget cap — an overlay teammate has no manifest row +/// at all, so its cap (if any) comes from the record's budget overrides via +/// [`CompanyRecord::effective_budget`], resolved by the caller. The overlay's +/// `name` is carried across (issue #1105): it is what +/// [`crate::metering::roster_display_names`] labels this teammate with +/// everywhere in the console, so +/// [`persona_prompt`](crate::company::prompt::persona_prompt) needs it to frame the +/// agent as the person the operator is addressing. Dropping it here — as this +/// did until #1105 — left the model knowing only its role, so it denied being +/// the name on its own DM header. +fn overlay_agent_to_manifest(overlay: &OverlayAgent) -> ManifestAgent { + ManifestAgent { + global: false, + id: overlay.id.clone(), + role: overlay.role.clone(), + name: Some(overlay.name.clone()), + description: overlay.description.clone(), + tier: None, + // An operator- or orchestrator-added teammate runs on the company's + // default harness. There is no console field to name one, and inventing + // a binding here would put a teammate on a harness nobody chose. + harness: None, + // Issue #661 / L5: carry the overlay's own per-teammate grant. An empty + // list here is unchanged behaviour — `agent_effective_grants` reads it as + // the standard company-wide grant, exactly as the hardcoded empty did. + // A non-empty list is intersected with `[tools].allow` by that same + // function below (narrow-only, never a widen). + tools: overlay.tools.clone(), + // Issue #176: an overlay teammate declares no delegation allowlist in + // this slice, so it carries today's behaviour — no hand-off tools wired. + // Opting overlays in needs a console write surface; see the follow-up. + delegates_to: Vec::new(), + context: None, + budget_usd_daily: None, + prompt: None, + prompt_files: Vec::new(), + prompt_files_resolved: Vec::new(), + classes: Vec::new(), + ledgers: None, + can_declare_ledgers: true, + } +} + +/// A minimal [`HarnessDeps`] for tests that only care about **workflow-tool +/// wiring**: which namespaces a `tool_call` can reach, and why the others cannot. +/// +/// Only the inputs [`workflow_tool_wiring`](crate::workflows::caps) actually +/// reads are parameters — the meter and plan (which resolve the capability +/// filter per company and spend) and the static filter itself. `search` is +/// pinned to `None`, because a deployment with no managed search backend is the +/// shape issue #874 is about. Everything else is the cheapest inert default, so +/// a test asserting on wiring does not have to name thirty fields that cannot +/// affect the answer. +/// +/// Shared rather than copied: the same fixture backs the runtime-level wiring +/// tests and the `tool-slugs` route test, so both ask about one deployment shape. +#[cfg(test)] +pub(crate) fn workflow_wiring_deps( + runtime: &crate::CompanyRuntime, + meter: Option>, + capabilities: toolbelt::CapabilityFilter, + plan: Option, +) -> HarnessDeps { + HarnessDeps { + ledgers: None, + ledger_registry: Default::default(), + provider: Arc::new(provider::MockProvider::default()), + provider_slug: "mock".to_string(), + serves: None, + context: runtime.context.clone(), + store: runtime.store.clone(), + meter, + workspace_root: std::env::temp_dir(), + workspace_git_enabled: false, + audit_root: std::env::temp_dir(), + model_override: None, + tasks: None, + artifacts: None, + skills: None, + skills_source_dir: None, + skills_registry: Arc::from([]), + mcp_servers: Vec::new(), + default_mcp_servers: Vec::new(), + facts: None, + events: None, + delegations: orchestrator::DelegationQueue::default(), + workflow_runner: orchestrator::WorkflowRunnerHandle::default(), + mcp_failures: mcp_probe::McpFailureQueue::default(), + pending_publishes: publish::PendingPublishQueue::default(), + workflow_refs: workflow_refs::WorkflowRefQueue::default(), + run_outputs: orchestrator::RunOutputCache::default(), + run_output_store: None, + workflow_revisions: None, + approval_requests: policy::ApprovalRequestQueue::default(), + secrets: None, + web_allowed_domains: Vec::new(), + capabilities, + workflow_source_dir: None, + plan, + media: None, + composio: None, + #[cfg(feature = "chargebee")] + chargebee: None, + #[cfg(feature = "paypal")] + paypal: None, + hosting: None, + // The staging shape in issue #874: `searchCredentialConfigured: false`. + search: None, + steer: crate::company::steer::InflightRegistry::default(), + run_supervisor: crate::runtime::RunSupervisor::default(), + delivery: None, + workspace: None, + repos: None, + repo_bindings: Vec::new(), + checkouts: repo::CheckoutLedger::default(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex as StdMutex; + + use async_trait::async_trait; + use tinyagents::harness::model::{ChatModel, ModelRequest, ModelResponse}; + + use crate::company::CompanyManifest; + use crate::harness::provider::MockProvider; + use crate::ports::UsageSample; + use crate::ports::types::{ + ChunkAddr, ChunkHit, ChunkMeta, CompanySummary, ContextChunk, LedgerEntry, + }; + // The two-level resolver. Test-only now: the roster build goes through + // `agent_scoped_grants`, and these tests assert the desk-less case still + // resolves identically to what shipped before desks could scope tools. + use crate::runtime::builder::agent_effective_grants; + + fn fp_entry(mode: Option<&str>, always: Option>) -> PolicyOverride { + use crate::ports::types::{Actor, ActorKind}; + PolicyOverride { + mode: mode.map(str::to_string), + always_approve: always.map(|v| v.into_iter().map(str::to_string).collect()), + set_by: Actor { + kind: ActorKind::User, + id: "user-1".to_string(), + }, + at_millis: 1_700_000_000_000, + } + } + + /// The fingerprint moves when the tier moves (issue #562). + /// + /// This is the assertion that keeps the feature from being a no-op. + /// `ApprovalPolicy` is built once per roster build, and `ensure` reuses the + /// cached roster unless a fingerprint changed — so if this returned a + /// constant, a console tier change would persist, return `204`, render as + /// applied, and be **silently ignored until the process restarted**. Every + /// other test in this change would still pass. + #[test] + fn the_policy_fingerprint_moves_when_the_tier_does() { + let none = policy_fingerprint(None); + let supervised = policy_fingerprint(Some(&fp_entry(Some("supervised"), None))); + let full = policy_fingerprint(Some(&fp_entry(Some("full"), None))); + + assert_ne!( + supervised, full, + "a tier change must move the fingerprint or the roster is never rebuilt" + ); + assert_ne!( + none, supervised, + "setting an override must move the fingerprint even when it names the \ + tier the manifest already had — the manifest can change under a rebuild" + ); + } + + /// An always-ask edit moves it too, including clearing the list. + /// + /// `always_approve` wins over every tier including `full`, so an edit that + /// did not rebuild would leave the gate enforcing a list the operator had + /// already changed — the failure mode is stricter *or* looser than what the + /// console shows, depending on the edit. + #[test] + fn the_policy_fingerprint_moves_when_the_always_ask_list_does() { + let absent = policy_fingerprint(Some(&fp_entry(Some("auto"), None))); + let empty = policy_fingerprint(Some(&fp_entry(Some("auto"), Some(vec![])))); + let one = policy_fingerprint(Some(&fp_entry(Some("auto"), Some(vec!["payment.send"])))); + let two = policy_fingerprint(Some(&fp_entry( + Some("auto"), + Some(vec!["payment.send", "filing.submit"]), + ))); + + assert_ne!( + absent, empty, + "clearing the list is not the same as not overriding it" + ); + assert_ne!(empty, one); + assert_ne!(one, two); + + // Order is part of the value: the list is the operator's own, not an + // accumulation of independent rows, so a reorder is a real edit. + let reordered = policy_fingerprint(Some(&fp_entry( + Some("auto"), + Some(vec!["filing.submit", "payment.send"]), + ))); + assert_ne!(two, reordered); + + // Length is folded in, so concatenation cannot collide. + let split = policy_fingerprint(Some(&fp_entry(Some("auto"), Some(vec!["a", "b"])))); + let joined = policy_fingerprint(Some(&fp_entry(Some("auto"), Some(vec!["ab"])))); + assert_ne!(split, joined); + } + + /// Issue #661 / L5: an overlay teammate's own `tools` grant flows into the + /// manifest shape `build_agent` consumes, and is INTERSECTED with the company + /// allow-list — narrow-only, never a widen. An empty grant is the standard + /// company-wide grant, exactly as the pre-L5 hardcoded empty was. + #[test] + fn overlay_agent_to_manifest_carries_the_tool_grant() { + let allow = vec!["docs.*".to_string(), "web".to_string()]; + + // A scoped overlay teammate: the grant is carried, then narrowed to what + // the company already allows. `payment.send` is NOT in `allow`, so the + // overlay cannot escalate to it — the security invariant. + let scoped = OverlayAgent { + id: "scoped".into(), + name: "Scoped".into(), + role: "Researcher".into(), + description: None, + tools: vec!["docs.*".into(), "payment.send".into()], + }; + let manifest = overlay_agent_to_manifest(&scoped); + assert_eq!( + manifest.tools, + vec!["docs.*".to_string(), "payment.send".to_string()], + "the overlay's own grant must reach the manifest shape" + ); + assert_eq!( + agent_effective_grants(&allow, &manifest.tools), + vec!["docs.*".to_string()], + "narrow-only: the un-allowed `payment.send` is intersected out" + ); + + // An empty overlay grant is the standard company-wide grant, unchanged. + let standard = OverlayAgent { + id: "std".into(), + name: "Std".into(), + role: "Generalist".into(), + description: None, + tools: Vec::new(), + }; + let manifest = overlay_agent_to_manifest(&standard); + assert!(manifest.tools.is_empty()); + assert_eq!( + agent_effective_grants(&allow, &manifest.tools), + allow, + "an empty grant falls back to the full company allow-list" + ); + } + + /// Issue #1105: the overlay's display name is the only place the operator's + /// chosen name exists, and the console shows it on the DM header, subtitle + /// and composer. Dropping it here left the persona framed from the role + /// alone, so the teammate denied being the person on its own header. + #[test] + fn overlay_agent_to_manifest_carries_the_display_name() { + let overlay = OverlayAgent { + id: "alex".into(), + name: "Alex".into(), + role: "Content Writer".into(), + description: None, + tools: Vec::new(), + }; + + let manifest = overlay_agent_to_manifest(&overlay); + assert_eq!(manifest.name.as_deref(), Some("Alex")); + // And it reaches the one place it has to: the persona the model reads. + let persona = crate::company::prompt::persona_prompt("Acme", &manifest); + assert!( + persona.contains("You are Alex, the Content Writer at Acme"), + "{persona}" + ); + } + + /// Issue #661 / L5: a grant edit changes the roster the harness must build, so + /// it has to move the overlay fingerprint — otherwise a re-grant would + /// persist, render as applied, and be silently ignored until the process + /// restarted, the same staleness the tier/skill fingerprints guard against. + #[test] + fn overlay_fingerprint_moves_on_a_tools_only_edit() { + let one = |tools: Vec| { + vec![OverlayAgent { + id: "a".into(), + name: "A".into(), + role: "r".into(), + description: None, + tools, + }] + }; + let standard = one(Vec::new()); + let scoped = one(vec!["docs.*".into()]); + let scoped_more = one(vec!["docs.*".into(), "email".into()]); + + assert_ne!( + overlay_fingerprint(&standard), + overlay_fingerprint(&scoped), + "adding a grant must move the fingerprint or the re-grant is ignored until restart" + ); + assert_ne!( + overlay_fingerprint(&scoped), + overlay_fingerprint(&scoped_more), + "widening the grant list must move it too" + ); + // Identical grants → identical fingerprint (no spurious rebuild). + assert_eq!( + overlay_fingerprint(&scoped), + overlay_fingerprint(&one(vec!["docs.*".into()])) + ); + } + + /// Attribution is deliberately NOT hashed. + /// + /// Re-setting the same tier writes a fresh `set_by`/`at_millis`. If those + /// moved the fingerprint, every such save would rebuild the roster and drop + /// live agent sessions for a change no agent can observe — the same reason + /// `budget_fingerprint` omits them. + #[test] + fn re_setting_the_same_tier_does_not_rebuild_the_roster() { + use crate::ports::types::{Actor, ActorKind}; + let first = fp_entry(Some("auto"), Some(vec!["payment.send"])); + let second = PolicyOverride { + set_by: Actor { + kind: ActorKind::User, + id: "a-different-admin".to_string(), + }, + at_millis: 1_900_000_000_000, + ..first.clone() + }; + assert_eq!( + policy_fingerprint(Some(&first)), + policy_fingerprint(Some(&second)), + "attribution must not move the fingerprint" + ); + } + + /// In-memory `ContextStore` so `OcMemory` has somewhere to land. + #[derive(Default)] + struct MockContext { + chunks: StdMutex>, + } + + #[async_trait] + impl ContextStore for MockContext { + async fn put(&self, _id: &CompanyId, chunk: ContextChunk) -> crate::Result { + let mut guard = self.chunks.lock().unwrap(); + let addr = ChunkAddr::new(format!("addr-{}", guard.len())); + guard.push((addr.clone(), chunk)); + Ok(addr) + } + async fn list(&self, _id: &CompanyId, prefix: &str) -> crate::Result> { + let guard = self.chunks.lock().unwrap(); + Ok(guard + .iter() + .filter(|(_, c)| c.label.starts_with(prefix)) + .map(|(addr, c)| ChunkMeta { + addr: addr.clone(), + label: c.label.clone(), + len: c.body.len(), + // The mock does not model store time; these tests exercise + // the harness, not the Brain's freshness stat. + stored_at_millis: 0, + }) + .collect()) + } + async fn peek( + &self, + _id: &CompanyId, + addr: &ChunkAddr, + _range: Option>, + ) -> crate::Result { + let guard = self.chunks.lock().unwrap(); + Ok(guard + .iter() + .find(|(a, _)| a == addr) + .map(|(_, c)| c.body.clone()) + .unwrap_or_default()) + } + async fn search( + &self, + _id: &CompanyId, + query: &str, + limit: usize, + ) -> crate::Result> { + let guard = self.chunks.lock().unwrap(); + Ok(guard + .iter() + .filter(|(_, c)| c.body.contains(query)) + .take(limit) + .map(|(addr, c)| ChunkHit { + addr: addr.clone(), + snippet: c.body.clone(), + score: 1.0, + }) + .collect()) + } + } + + /// `CompanyStore` that records what the cost hook appends. + #[derive(Default)] + struct RecordingStore { + ledger: StdMutex>, + } + + #[async_trait] + impl CompanyStore for RecordingStore { + async fn load(&self, _id: &CompanyId) -> crate::Result> { + Ok(None) + } + async fn save(&self, _record: &CompanyRecord) -> crate::Result<()> { + Ok(()) + } + async fn list(&self) -> crate::Result> { + Ok(Vec::new()) + } + async fn append_ledger(&self, _id: &CompanyId, entry: LedgerEntry) -> crate::Result<()> { + self.ledger.lock().unwrap().push(entry); + Ok(()) + } + } + + /// Records usage samples so a zero-usage turn can be asserted inert. + #[derive(Default)] + struct RecordingMeter { + samples: StdMutex>, + } + + #[async_trait] + impl UsageMeter for RecordingMeter { + async fn record(&self, _company: &CompanyId, sample: &UsageSample) -> crate::Result<()> { + self.samples.lock().unwrap().push(sample.clone()); + Ok(()) + } + /// Honours `since_millis`, per the port contract ("every sample at or + /// after `since_millis`"). The per-agent daily cap (issue #304) is a + /// windowed read, so a double that returned everything regardless would + /// make the day-rollover test pass against any boundary the code + /// computed — including none at all. + async fn query(&self, _company: &CompanyId, since: u64) -> crate::Result> { + Ok(self + .samples + .lock() + .unwrap() + .iter() + .filter(|sample| sample.at_millis >= since) + .cloned() + .collect()) + } + } + + /// A meter whose reads always fail — for the dispatch gate's fail-open pin. + struct FailingMeter; + + #[async_trait] + impl UsageMeter for FailingMeter { + async fn record(&self, _company: &CompanyId, _sample: &UsageSample) -> crate::Result<()> { + Ok(()) + } + async fn query( + &self, + _company: &CompanyId, + _since: u64, + ) -> crate::Result> { + Err(OpenCompanyError::Store("meter unavailable".into())) + } + } + + fn manifest() -> CompanyManifest { + toml::from_str( + r#" +[company] +name = "Acme" + +[policy] +mode = "full" + +[[agent]] +id = "ceo" +role = "Chief Executive" +description = "Sets direction." + +[[agent]] +id = "engineer" +role = "Engineer" +description = "Builds the product." +"#, + ) + .expect("valid manifest") + } + + fn record() -> CompanyRecord { + CompanyRecord { + id: CompanyId::new("acme"), + manifest: manifest(), + ledger: Vec::new(), + lifecycle: "running".to_string(), + setup: None, + overlay_agents: Vec::new(), + overlay_desk_members: Vec::new(), + overlay_desk_order: Vec::new(), + overlay_desks: Vec::new(), + overlay_workflows: Vec::new(), + overlay_budgets: Vec::new(), + overlay_policy: None, + overlay_desk_tools: Default::default(), + disabled_workflows: Vec::new(), + template_provenance: None, + } + } + + struct Fixture { + deps: HarnessDeps, + store: Arc, + meter: Arc, + _dir: tempfile::TempDir, + } + + fn fixture() -> Fixture { + let dir = tempfile::tempdir().expect("tempdir"); + let store = Arc::new(RecordingStore::default()); + let meter = Arc::new(RecordingMeter::default()); + Fixture { + deps: HarnessDeps { + ledgers: None, + ledger_registry: Default::default(), + provider: Arc::new(MockProvider::new("mock: ")), + provider_slug: "mock".to_string(), + serves: None, + context: Arc::new(MockContext::default()), + store: store.clone(), + meter: Some(meter.clone()), + workspace_root: dir.path().to_path_buf(), + workspace_git_enabled: false, + audit_root: dir.path().to_path_buf(), + model_override: None, + tasks: None, + artifacts: None, + skills: None, + skills_source_dir: None, + skills_registry: std::sync::Arc::from([]), + default_mcp_servers: Vec::new(), + mcp_servers: Vec::new(), + facts: None, + events: None, + delegations: DelegationQueue::default(), + workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), + mcp_failures: McpFailureQueue::default(), + pending_publishes: crate::harness::publish::PendingPublishQueue::default(), + workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), + run_outputs: crate::harness::orchestrator::RunOutputCache::default(), + run_output_store: None, + workflow_revisions: None, + approval_requests: ApprovalRequestQueue::default(), + secrets: None, + web_allowed_domains: Vec::new(), + capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, + workflow_source_dir: None, + plan: None, + media: None, + composio: None, + #[cfg(feature = "chargebee")] + chargebee: None, + #[cfg(feature = "paypal")] + paypal: None, + hosting: None, + steer: crate::company::steer::InflightRegistry::default(), + run_supervisor: crate::runtime::RunSupervisor::default(), + delivery: None, + search: None, + workspace: None, + repos: None, + repo_bindings: Vec::new(), + checkouts: crate::harness::repo::CheckoutLedger::default(), + }, + store, + meter, + _dir: dir, + } + } + + #[tokio::test] + async fn roster_builds_every_manifest_agent() { + let fx = fixture(); + let roster = + build_roster(&record(), &fx.deps, &[], &HashMap::new()).expect("roster builds"); + let ids: Vec<_> = roster.iter().map(|a| a.agent_id.as_str()).collect(); + assert_eq!(ids, vec!["ceo", "engineer"]); + assert_eq!(roster[0].role, "Chief Executive"); + } + + /// Context routing: the resolution that feeds a persona, and the fingerprint + /// that decides whether an edit reaches the next turn. + mod routed_context { + use super::*; + use crate::ports::workspace::{NodeKind, WorkspaceNode, WorkspaceOrigin}; + + fn docs(entries: &[(&str, &[(&str, &str)])]) -> HashMap> { + entries + .iter() + .map(|(agent, documents)| { + ( + (*agent).to_string(), + documents + .iter() + .map(|(p, b)| ((*p).to_string(), (*b).to_string())) + .collect(), + ) + }) + .collect() + } + + /// The property the whole axis exists for. The routing table is manifest + /// data and does not move when an operator edits a note, so a + /// name-only hash would leave the edit invisible until a restart. + #[test] + fn the_fingerprint_moves_when_a_documents_body_changes() { + let before = routed_context_fingerprint(&docs(&[("ceo", &[("BRIEF.md", "old")])])); + let after = routed_context_fingerprint(&docs(&[("ceo", &[("BRIEF.md", "new")])])); + assert_ne!( + before, after, + "an edited routed note must rebuild the roster" + ); + } + + /// A `HashMap` has no order, so an order-sensitive hash would drop every + /// live agent session on a rebuild that changed nothing. + #[test] + fn the_fingerprint_is_stable_across_map_iteration_order() { + let one = docs(&[ + ("ceo", &[("BRIEF.md", "b")]), + ("engineer", &[("CLAIMS.md", "c")]), + ]); + let two = docs(&[ + ("engineer", &[("CLAIMS.md", "c")]), + ("ceo", &[("BRIEF.md", "b")]), + ]); + assert_eq!( + routed_context_fingerprint(&one), + routed_context_fingerprint(&two) + ); + } + + /// Renaming a document is a real change even when its text is identical: + /// the persona quotes the path as the section heading. + #[test] + fn the_fingerprint_moves_when_a_document_is_renamed() { + let before = routed_context_fingerprint(&docs(&[("ceo", &[("BRIEF.md", "same")])])); + let after = routed_context_fingerprint(&docs(&[("ceo", &[("GOAL.md", "same")])])); + assert_ne!(before, after); + } + + /// A company with no workspace store keeps a stable fingerprint and never + /// rebuilds on this axis — the pre-routing behaviour exactly. + #[tokio::test] + async fn no_workspace_store_resolves_to_nothing() { + let fx = fixture(); + assert!(fx.deps.workspace.is_none(), "fixture has no store wired"); + + let pool = HarnessPool::new(); + let routed = pool.resolve_routed_context(&record(), &fx.deps, &[]).await; + assert!(routed.is_empty(), "{routed:?}"); + assert_eq!( + routed_context_fingerprint(&routed), + routed_context_fingerprint(&HashMap::new()), + "a company that routes nothing must not rebuild on this axis" + ); + } + + /// The real path: a routed document that exists in the tree is read and + /// keyed to the agent whose manifest asked for it. + #[tokio::test] + async fn a_routed_document_is_resolved_per_agent() { + let dir = tempfile::tempdir().expect("tempdir"); + let ws: Arc = + Arc::new(crate::store::FsOps::new(dir.path())); + let company = CompanyId::new("acme"); + ws.create( + &company, + &WorkspaceNode { + id: "n-brief".to_string(), + name: "BRIEF.md".to_string(), + kind: NodeKind::File, + parent_id: None, + updated_at_millis: 1, + created_by: WorkspaceOrigin::Operator, + updated_by: WorkspaceOrigin::Operator, + mime: None, + size: None, + sha256: None, + }, + Some("What the company established."), + ) + .await + .expect("create"); + + let mut fx = fixture(); + fx.deps.workspace = Some(ws); + + let pool = HarnessPool::new(); + let routed = pool.resolve_routed_context(&record(), &fx.deps, &[]).await; + + // Both fixture agents default to the `reasoning` row, which routes + // BRIEF — so both resolve it, and neither invents the notes that do + // not exist in the tree. + for agent in ["ceo", "engineer"] { + let documents = routed + .get(agent) + .unwrap_or_else(|| panic!("no routed documents for {agent}: {routed:?}")); + assert_eq!( + documents, + &vec![( + "BRIEF.md".to_string(), + "What the company established.".to_string() + )], + "{agent}" + ); + } + } + } + + /// The roster builds end-to-end with the skill read surface wired: the + /// effective set materializes, the read tools build, and the catalogue folds + /// into the persona — all without error — and the scratch tree lands under + /// the agent's workspace root. + #[tokio::test] + async fn roster_builds_with_skill_surface_wired() { + let dir = tempfile::tempdir().expect("tempdir"); + let source = tempfile::tempdir().expect("source"); + let skill_dir = source.path().join("skills").join("web-research"); + std::fs::create_dir_all(&skill_dir).unwrap(); + std::fs::write( + skill_dir.join("SKILL.md"), + "---\nname: Web Research\ndescription: Answer a question\n---\n\n# Web Research\n", + ) + .unwrap(); + + let deps = HarnessDeps { + ledgers: None, + ledger_registry: Default::default(), + provider: Arc::new(MockProvider::new("mock: ")), + provider_slug: "mock".to_string(), + serves: None, + context: Arc::new(MockContext::default()), + store: Arc::new(RecordingStore::default()), + meter: None, + workspace_root: dir.path().to_path_buf(), + workspace_git_enabled: false, + audit_root: dir.path().to_path_buf(), + model_override: None, + tasks: None, + artifacts: None, + skills: None, + skills_source_dir: Some(source.path().to_path_buf()), + skills_registry: std::sync::Arc::from([]), + default_mcp_servers: Vec::new(), + mcp_servers: Vec::new(), + facts: None, + events: None, + delegations: DelegationQueue::default(), + workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), + mcp_failures: McpFailureQueue::default(), + pending_publishes: crate::harness::publish::PendingPublishQueue::default(), + workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), + run_outputs: crate::harness::orchestrator::RunOutputCache::default(), + run_output_store: None, + workflow_revisions: None, + approval_requests: ApprovalRequestQueue::default(), + secrets: None, + web_allowed_domains: Vec::new(), + capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, + workflow_source_dir: None, + plan: None, + media: None, + composio: None, + #[cfg(feature = "chargebee")] + chargebee: None, + #[cfg(feature = "paypal")] + paypal: None, + hosting: None, + steer: crate::company::steer::InflightRegistry::default(), + run_supervisor: crate::runtime::RunSupervisor::default(), + delivery: None, + search: None, + workspace: None, + repos: None, + repo_bindings: Vec::new(), + checkouts: crate::harness::repo::CheckoutLedger::default(), + }; + + let roster = build_roster(&record(), &deps, &[], &HashMap::new()) + .expect("roster builds with skills"); + assert_eq!(roster.len(), 2); + // The scratch skill tree was materialized for the first roster agent. + assert!( + dir.path() + .join("acme") + .join("ceo") + .join("skill-catalog") + .join("skills") + .join("web-research") + .join("SKILL.md") + .is_file(), + "the effective skill bundle should be materialized under the agent workspace" + ); + } + + /// Issue #71 — an operator/orchestrator-added overlay teammate is promoted + /// into a real, addressable roster agent (not just a console row). + #[tokio::test] + async fn overlay_agent_is_built_as_a_real_roster_agent() { + let fx = fixture(); + let mut rec = record(); + rec.overlay_agents.push(OverlayAgent { + id: "growth".into(), + name: "Jamie".into(), + role: "Growth Lead".into(), + description: Some("Owns acquisition experiments.".into()), + tools: Vec::new(), + }); + + let roster = build_roster(&rec, &fx.deps, &[], &HashMap::new()).expect("roster builds"); + let ids: Vec<_> = roster.iter().map(|a| a.agent_id.as_str()).collect(); + assert_eq!(ids, vec!["ceo", "engineer", "growth"], "got {ids:?}"); + let overlay_agent = roster + .iter() + .find(|a| a.agent_id == "growth") + .expect("overlay teammate present in roster"); + assert_eq!(overlay_agent.role, "Growth Lead"); + } + + /// A manifest agent always wins an id collision with an overlay teammate — + /// the version-controlled roster is authoritative. + #[tokio::test] + async fn overlay_agent_id_colliding_with_manifest_agent_is_skipped() { + let fx = fixture(); + let mut rec = record(); + rec.overlay_agents.push(OverlayAgent { + id: "ceo".into(), + name: "Impostor".into(), + role: "Shadow CEO".into(), + description: None, + tools: Vec::new(), + }); + + let roster = build_roster(&rec, &fx.deps, &[], &HashMap::new()).expect("roster builds"); + let ids: Vec<_> = roster.iter().map(|a| a.agent_id.as_str()).collect(); + assert_eq!( + ids, + vec!["ceo", "engineer"], + "the manifest agent wins the id collision, not a duplicate" + ); + assert_eq!( + roster[0].role, "Chief Executive", + "the manifest role survives, not the overlay's" + ); + } + + /// Issue #686, end to end: the orchestrator adds a teammate whose display + /// name slugs onto a **manifest** agent's id, and the teammate still shows + /// up in the built roster. + /// + /// This is the failure the suffix exists to prevent, and it only became + /// reachable when ids started coming from names. `add_agent`'s duplicate + /// guard compares overlay *names*, so "Engineer" sails past it; an + /// unsuffixed `engineer` would then be skipped by + /// [`build_roster`](super::build_roster) as already claimed by the manifest + /// — saved to the record, never materialised, no error anywhere. + #[tokio::test] + async fn a_tool_added_teammate_colliding_with_a_manifest_id_still_joins_the_roster() { + use openhuman_core::openhuman::tools::Tool; + + use crate::harness::orchestrator::unscoped_add_agent; + + /// A `CompanyStore` that actually holds the record, unlike + /// `RecordingStore` — `add_agent` has to load what it saves. + struct SeededStore(StdMutex); + + #[async_trait] + impl CompanyStore for SeededStore { + async fn load(&self, _id: &CompanyId) -> crate::Result> { + Ok(Some(self.0.lock().unwrap().clone())) + } + async fn save(&self, record: &CompanyRecord) -> crate::Result<()> { + *self.0.lock().unwrap() = record.clone(); + Ok(()) + } + async fn list(&self) -> crate::Result> { + Ok(Vec::new()) + } + async fn append_ledger( + &self, + _id: &CompanyId, + _entry: LedgerEntry, + ) -> crate::Result<()> { + Ok(()) + } + } + + let fx = fixture(); + let company = CompanyId::new("acme"); + let store = Arc::new(SeededStore(StdMutex::new(record()))); + let tool = unscoped_add_agent(company.clone(), store.clone()); + + let result = tool + .execute(serde_json::json!({ "name": "Engineer", "role": "Platform" })) + .await + .expect("execute"); + assert!( + !result.is_error, + "the name guard compares overlay names only" + ); + assert!( + result.text().contains("engineer_2"), + "the orchestrator has to learn the id it can address: {}", + result.text() + ); + + let saved = store.load(&company).await.unwrap().expect("record"); + assert_eq!(saved.overlay_agents[0].id, "engineer_2"); + + let roster = build_roster(&saved, &fx.deps, &[], &HashMap::new()).expect("roster builds"); + let ids: Vec<_> = roster.iter().map(|a| a.agent_id.as_str()).collect(); + assert_eq!( + ids, + vec!["ceo", "engineer", "engineer_2"], + "a suffixed id materialises; an unsuffixed one would vanish here" + ); + } + + /// Issue #551: a roster rebuild writes nothing to the workspace. + /// + /// This used to be the feature's second provisioning seam — a teammate + /// added at runtime (a manifest edit, the console's `add_member`, the + /// orchestrator's `add_agent`) reaches the harness as a moved overlay + /// fingerprint, and the folder was minted here. A member folder is no + /// longer a function of the roster, so joining one is no longer an event + /// the tree records: the folder appears when the teammate first produces + /// something, and the two system roots come from boot. + /// + /// Pinned as a test because a rebuild that quietly resumed writing would + /// re-fill the tree with empty folders for teammates who have done nothing + /// — exactly the noise this change removed. + #[tokio::test] + async fn a_roster_rebuild_writes_nothing_to_the_workspace() { + let dir = tempfile::tempdir().expect("tempdir"); + let ws: Arc = + Arc::new(crate::store::FsOps::new(dir.path())); + let mut fx = fixture(); + fx.deps.workspace = Some(ws.clone()); + + let mut rec = record(); + let pool = HarnessPool::new(); + pool.ensure(&rec, &fx.deps).await.expect("first ensure"); + assert!( + ws.is_empty(&rec.id).await.expect("is_empty"), + "the roster build touched the workspace" + ); + + // The runtime-added teammate. The overlay fingerprint moves, so this + // `ensure` takes the rebuild path rather than the cached fast path. + rec.overlay_agents.push(OverlayAgent { + id: "designer".into(), + name: "Dana".into(), + role: "Designer".into(), + description: None, + tools: Vec::new(), + }); + pool.ensure(&rec, &fx.deps).await.expect("second ensure"); + + assert!( + ws.is_empty(&rec.id).await.expect("is_empty"), + "the rebuild minted a folder for a teammate that has produced nothing" + ); + + // …and the folder the teammate *does* get is the one it earns by + // producing something, minted through the lazy seam instead. + let minted = crate::company::workspace_scaffold::ensure_agent_folder( + ws.as_ref(), + &rec.id, + "designer", + ) + .await + .expect("mint"); + let tree = ws.tree(&rec.id).await.expect("tree"); + let mut names: Vec<&str> = tree.iter().map(|n| n.name.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, vec!["Agents", "designer"]); + assert_eq!( + tree.iter().find(|n| n.id == minted).unwrap().created_by, + crate::ports::WorkspaceOrigin::Agent { + id: "designer".to_string() + }, + ); + } + + #[tokio::test] + async fn run_executes_a_turn_on_the_openhuman_runtime() { + let fx = fixture(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + + let reply = pool + .run(&rec.id, "ceo", "hello-marker", &fx.deps, None) + .await + .expect("turn runs") + .reply; + + assert!( + reply.contains("hello-marker"), + "reply should echo the prompt through the agent: {reply:?}" + ); + } + + #[tokio::test] + async fn run_stores_outcomes_and_injects_them_into_later_turns() { + let fx = fixture(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + + // Cold store: nothing to inject on the first turn. + let first = pool + .run(&rec.id, "ceo", "alpha task", &fx.deps, None) + .await + .expect("first turn") + .reply; + assert!( + !first.contains("Relevant prior work"), + "a cold turn injects nothing: {first:?}" + ); + + // The outcome was written back under the task-outcome prefix. + let stored = fx + .deps + .context + .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) + .await + .unwrap(); + assert_eq!(stored.len(), 1, "the first turn stores its outcome"); + + // Second turn: the prior outcome (its body contains "alpha") is + // retrieved and injected, so the agent sees the preamble. + let second = pool + .run(&rec.id, "ceo", "alpha", &fx.deps, None) + .await + .expect("second turn") + .reply; + assert!( + second.contains("Relevant prior work"), + "the second turn injects the retrieved outcome: {second:?}" + ); + + let stored = fx + .deps + .context + .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) + .await + .unwrap(); + assert_eq!(stored.len(), 2, "the second turn stores its outcome too"); + } + + /// A pool serving one named harness builds only the agents bound to it. + /// + /// This is what makes one-pool-per-harness affordable: without the filter a + /// ten-agent roster on three harnesses would stand up thirty live agents, + /// each holding a model client, to use ten. + #[tokio::test] + async fn a_scoped_pool_builds_only_the_agents_it_serves() { + let fx = fixture(); + let rec = record(); + assert!( + rec.manifest.agents.len() >= 2, + "the fixture must have someone to leave out" + ); + + // Unfiltered: the whole roster, exactly as before this field existed. + let pool = HarnessPool::new(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + let all = pool.agent_ids(&rec.id).await; + assert_eq!(all.len(), rec.manifest.agents.len()); + + // Scoped to one agent: only that one is built. + let mut scoped = fixture(); + scoped.deps.serves = Some(HashSet::from(["ceo".to_string()])); + let pool = HarnessPool::new(); + pool.ensure(&rec, &scoped.deps).await.expect("ensure"); + assert_eq!(pool.agent_ids(&rec.id).await, vec!["ceo".to_string()]); + } + + #[tokio::test] + async fn ensure_is_idempotent() { + let fx = fixture(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("first ensure"); + pool.ensure(&rec, &fx.deps).await.expect("second ensure"); + assert_eq!(pool.resident_companies().await, 1); + } + + #[tokio::test] + async fn turns_are_serialised_and_history_survives() { + let fx = fixture(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + + pool.run(&rec.id, "ceo", "first", &fx.deps, None) + .await + .expect("first turn"); + let second = pool + .run(&rec.id, "ceo", "second", &fx.deps, None) + .await + .expect("second turn") + .reply; + + assert!(second.contains("second")); + } + + /// Issue #416 — a confined turn reaches the company's memory neither on the + /// way in nor on the way out. + /// + /// The control half is what makes this a test rather than an assertion of + /// absence: the SAME message on the ordinary roster path pulls the seeded + /// chunk into the prompt (the mock provider echoes what it was sent, so the + /// injection is visible in the reply), and writes the turn back. The + /// confined path does neither, from the same store, in the same test. + #[tokio::test] + async fn a_confined_turn_neither_reads_nor_writes_company_memory() { + let context = Arc::new(MockContext::default()); + let mut fx = fixture(); + fx.deps.context = context.clone(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + + // A prior outcome sitting in the company's memory. The mock store + // matches a chunk whose BODY contains the query, and retrieve→inject + // queries with the whole message — so a body built around the message is + // what a hit looks like here. + let question = "why did it fail"; + context + .put( + &rec.id, + ContextChunk { + label: "prior/outcome".into(), + body: format!("SECRET-PAYROLL-REVIEW: {question} on Monday"), + }, + ) + .await + .expect("seed the company's memory"); + let seeded = context.chunks.lock().unwrap().len(); + + // Control: the ordinary path injects the hit and writes the turn back. + let ordinary = pool + .run(&rec.id, "ceo", question, &fx.deps, None) + .await + .expect("the ordinary turn runs") + .reply; + assert!( + ordinary.contains("SECRET-PAYROLL-REVIEW"), + "the retrieve→inject step must be live for this test to mean anything: {ordinary}" + ); + assert!( + context.chunks.lock().unwrap().len() > seeded, + "the ordinary path writes its outcome back to company memory" + ); + + let before_confined = context.chunks.lock().unwrap().len(); + let confined = pool + .run_confined( + &rec.id, + "Acme", + question, + &fx.deps, + Some("workflow-copilot:weekly_report"), + &confine::Confinement::workflow("weekly_report"), + ) + .await + .expect("the confined turn runs") + .reply; + + assert!( + confined.contains(question), + "the confined turn still answers the question it was asked: {confined}" + ); + assert!( + !confined.contains("SECRET-PAYROLL-REVIEW"), + "a confined turn must not be handed company memory: {confined}" + ); + assert_eq!( + context.chunks.lock().unwrap().len(), + before_confined, + "a confined turn must leave nothing behind for a later turn to retrieve" + ); + } + + /// The confined agent is not on the roster, so nothing can address it: a + /// dispatch, a desk hand-off or a `chat` naming it is an unknown agent, the + /// same as any other name that is not a teammate. + #[tokio::test] + async fn the_confined_agent_is_not_addressable() { + let fx = fixture(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + + let err = pool + .run(&rec.id, confine::CONFINED_AGENT_ID, "hi", &fx.deps, None) + .await + .expect_err("the confined agent is not a roster agent"); + assert!( + matches!(err, OpenCompanyError::InvalidRequest(_)), + "{err:?}" + ); + } + + #[tokio::test] + async fn unknown_agent_is_invalid_request() { + let fx = fixture(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + + let err = pool + .run(&rec.id, "nobody", "hi", &fx.deps, None) + .await + .expect_err("unknown agent rejected"); + assert!( + matches!(err, OpenCompanyError::InvalidRequest(_)), + "{err:?}" + ); + } + + #[tokio::test] + async fn unknown_company_is_not_found() { + let fx = fixture(); + let pool = HarnessPool::new(); + let err = pool + .run(&CompanyId::new("ghost"), "ceo", "hi", &fx.deps, None) + .await + .expect_err("unknown company rejected"); + assert!( + matches!(err, OpenCompanyError::CompanyNotFound(_)), + "{err:?}" + ); + } + + // --- Workspace-ensure log edge-triggering (issue #449) ------------------- + + /// The whole transition table, exhaustively: a broken volume must produce + /// one error line and then nothing, and a recovery must be announced once. + #[test] + fn workspace_report_is_edge_triggered() { + let mut failing: HashSet<&str> = HashSet::new(); + + // First failure speaks. + assert_eq!( + workspace_report(&mut failing, &"a", true), + WorkspaceReport::Failed + ); + // Every repeat is silent — this is the flood #449 is about. + for _ in 0..100 { + assert_eq!( + workspace_report(&mut failing, &"a", true), + WorkspaceReport::StillFailing + ); + } + // Recovery speaks exactly once. + assert_eq!( + workspace_report(&mut failing, &"a", false), + WorkspaceReport::Recovered + ); + assert_eq!( + workspace_report(&mut failing, &"a", false), + WorkspaceReport::StillHealthy + ); + // A healthy agent that was never failing says nothing on its first + // attempt either — a working workspace has never been worth a line. + assert_eq!( + workspace_report(&mut failing, &"never-failed", false), + WorkspaceReport::StillHealthy + ); + // And it can fail again later: the edge re-arms. + assert_eq!( + workspace_report(&mut failing, &"a", true), + WorkspaceReport::Failed + ); + + assert!( + WorkspaceReport::StillFailing.is_silent() && WorkspaceReport::StillHealthy.is_silent(), + "only the repeats are silent" + ); + assert!( + !WorkspaceReport::Failed.is_silent() && !WorkspaceReport::Recovered.is_silent(), + "both edges must be reported" + ); + } + + /// Two agents interleaved: one failing, one healthy. Each key's edge is its + /// own — a second agent's failure must not be swallowed by the first's, and + /// a second agent's recovery must not clear the first's failure. + #[test] + fn workspace_report_tracks_each_key_separately() { + let mut failing: HashSet<&str> = HashSet::new(); + + assert_eq!( + workspace_report(&mut failing, &"ceo", true), + WorkspaceReport::Failed + ); + // A different agent failing is its own first failure, not a repeat. + assert_eq!( + workspace_report(&mut failing, &"engineer", true), + WorkspaceReport::Failed + ); + assert_eq!( + workspace_report(&mut failing, &"ceo", true), + WorkspaceReport::StillFailing + ); + // One recovers; the other stays failing and stays silent. + assert_eq!( + workspace_report(&mut failing, &"engineer", false), + WorkspaceReport::Recovered + ); + assert_eq!( + workspace_report(&mut failing, &"ceo", true), + WorkspaceReport::StillFailing + ); + assert_eq!( + workspace_report(&mut failing, &"ceo", false), + WorkspaceReport::Recovered + ); + assert!(failing.is_empty(), "a recovered key leaves no residue"); + } + + /// The real dispatch path against a workspace root that cannot hold a + /// directory, driven through [`HarnessPool::run`] rather than the helper. + /// + /// The root is pointed at a **file**, which makes `create_dir_all` fail + /// deterministically on every platform (`ENOTDIR` / its Windows equivalent) + /// without needing permission bits a CI root user would ignore. + /// + /// Asserts the reporting state, not the log text: this test binary already + /// installs a global `tracing` subscriber elsewhere + /// (`runtime::workflow_scheduler`) and asserts it wins that race, so a + /// second global capture here would make whichever test lost panic. The + /// state is what decides whether a line is emitted, so pinning it pins the + /// line count — three dispatches, one report. + #[tokio::test] + async fn a_broken_workspace_root_reports_once_across_repeated_dispatches() { + let dir = tempfile::tempdir().expect("tempdir"); + // A regular file where the workspace tree is expected. + let not_a_dir = dir.path().join("workspace-root"); + std::fs::write(¬_a_dir, b"this is a file, not a directory").unwrap(); + + let mut fx = fixture(); + fx.deps.workspace_root = not_a_dir.clone(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + + // Sanity: the condition really is a hard, repeatable failure. + assert!( + build::ensure_agent_workspace(¬_a_dir, &rec.id, "ceo").is_err(), + "the test root must actually be unusable, or this proves nothing" + ); + + for turn in 0..3 { + pool.run(&rec.id, "ceo", "hi", &fx.deps, None) + .await + .unwrap_or_else(|e| panic!("turn {turn} still runs without a workspace: {e:?}")); + } + + // The turns ran — a missing workspace is not fatal, which #449 does not + // change — and the failure is recorded exactly once. + let failing = pool.workspace_failures.lock().unwrap(); + assert_eq!( + failing.len(), + 1, + "one failing agent, tracked once, however many turns it takes" + ); + assert!(failing.contains(&(rec.id.clone(), "ceo".to_string()))); + drop(failing); + + // The next dispatch after the first is silent: only turn 1 spoke. + assert_eq!( + pool.note_workspace_attempt(&rec.id, "ceo", true), + WorkspaceReport::StillFailing, + "dispatches after the first must not re-emit the error" + ); + // And when the volume comes back, one line says so. + assert_eq!( + pool.note_workspace_attempt(&rec.id, "ceo", false), + WorkspaceReport::Recovered + ); + } + + /// Pins the documented inert-metering contract: until the provider reports + /// usage, a turn writes neither a ledger entry nor a usage sample. + #[tokio::test] + async fn zero_usage_turn_writes_nothing() { + let fx = fixture(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("ensure"); + pool.run(&rec.id, "ceo", "hi", &fx.deps, None) + .await + .expect("turn"); + + assert!(fx.store.ledger.lock().unwrap().is_empty()); + assert!(fx.meter.samples.lock().unwrap().is_empty()); + } + + // --- Empty-response turn wrapper ---------------------------------------- + + /// A model that plays back a scripted sequence of outcomes, one per + /// [`invoke`](ChatModel::invoke) call, so the empty-response retry wrapper can + /// be driven deterministically. `Ok("")` is the transient empty class (the + /// harness turn raises the empty-response error on a blank assistant reply); + /// `Err(_)` is a hard error. + struct ScriptedProvider { + script: StdMutex>>, + calls: std::sync::atomic::AtomicUsize, + } + + impl ScriptedProvider { + fn new(outcomes: Vec>) -> Self { + Self { + script: StdMutex::new(outcomes.into_iter().collect()), + calls: std::sync::atomic::AtomicUsize::new(0), + } + } + } + + #[async_trait] + impl ChatModel<()> for ScriptedProvider { + async fn invoke( + &self, + _state: &(), + _request: ModelRequest, + ) -> tinyagents::Result { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + match self.script.lock().unwrap().pop_front() { + Some(Ok(reply)) => Ok(ModelResponse::assistant(reply)), + Some(Err(err)) => Err(tinyagents::TinyAgentsError::Model(err)), + None => Ok(ModelResponse::assistant("exhausted")), + } + } + } + + impl HarnessModel for ScriptedProvider { + fn telemetry_provider_id(&self) -> String { + "scripted".to_string() + } + } + + /// Build a single [`CompanyAgent`] over a scripted provider so the wrapper can + /// be exercised directly (its retry logic is the unit under test). + fn scripted_agent(outcomes: Vec>) -> (Arc, HarnessDeps) { + let dir = tempfile::tempdir().expect("tempdir"); + let deps = HarnessDeps { + ledgers: None, + ledger_registry: Default::default(), + provider: Arc::new(ScriptedProvider::new(outcomes)), + provider_slug: "scripted".to_string(), + serves: None, + context: Arc::new(MockContext::default()), + store: Arc::new(RecordingStore::default()), + meter: None, + workspace_root: dir.path().to_path_buf(), + workspace_git_enabled: false, + audit_root: dir.path().to_path_buf(), + model_override: None, + tasks: None, + artifacts: None, + skills: None, + skills_source_dir: None, + skills_registry: std::sync::Arc::from([]), + default_mcp_servers: Vec::new(), + mcp_servers: Vec::new(), + facts: None, + events: None, + delegations: DelegationQueue::default(), + workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), + mcp_failures: McpFailureQueue::default(), + pending_publishes: crate::harness::publish::PendingPublishQueue::default(), + workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), + run_outputs: crate::harness::orchestrator::RunOutputCache::default(), + run_output_store: None, + workflow_revisions: None, + approval_requests: ApprovalRequestQueue::default(), + secrets: None, + web_allowed_domains: Vec::new(), + capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, + workflow_source_dir: None, + plan: None, + media: None, + composio: None, + #[cfg(feature = "chargebee")] + chargebee: None, + #[cfg(feature = "paypal")] + paypal: None, + hosting: None, + steer: crate::company::steer::InflightRegistry::default(), + run_supervisor: crate::runtime::RunSupervisor::default(), + delivery: None, + search: None, + workspace: None, + repos: None, + repo_bindings: Vec::new(), + checkouts: crate::harness::repo::CheckoutLedger::default(), + }; + let roster = build_roster(&record(), &deps, &[], &HashMap::new()).expect("roster"); + // Keep the tempdir alive for the agent's workspace by leaking it into the + // test's lifetime — the process ends the test anyway. + std::mem::forget(dir); + (roster.into_iter().next().expect("one agent"), deps) + } + + /// Empty first, real reply on retry → the wrapper returns the recovered reply + /// and reports two attempts' usage (so both burnt attempts can be metered). + #[tokio::test] + async fn turn_wrapper_retries_empty_then_recovers() { + let (agent, _deps) = scripted_agent(vec![Ok(String::new()), Ok("recovered".into())]); + let (outcome, usages) = agent.run("hi").await.expect("wrapper recovers"); + assert!( + outcome.reply.contains("recovered"), + "got {:?}", + outcome.reply + ); + assert_eq!(usages.len(), 2, "both attempts' usage is returned"); + } + + /// Issue #111 retry-guard edge: when a steer already pends and the first + /// attempt is the transient empty class, the one-shot retry is SKIPPED — so a + /// cancel/pause issued before any text can't silently restart the work. The + /// steered-empty turn therefore makes EXACTLY ONE attempt. + #[tokio::test] + async fn steered_empty_turn_makes_exactly_one_attempt() { + // Attempt 1 is empty; a normal `run` would retry and consume the second + // script entry. With a steer pending, the retry must not fire. + let (agent, _deps) = scripted_agent(vec![Ok(String::new()), Ok("second".into())]); + let control = SteerControl::new(); + control.request(SteerAction::Cancel); + let (_outcome, usages) = agent + .run_with_steer("hi", Some(&control), None, None) + .await + .expect("runs"); + assert_eq!( + usages.len(), + 1, + "a steered empty turn does NOT retry — exactly one attempt" + ); + } + + // Note: the *installation* of the steer stop-hook can't be observed from the + // provider — the tinyagents adapter snapshots the hooks at turn entry and the + // provider call may run on a spawned task where the task-local isn't + // inherited. The steer mechanism is instead proven end-to-end by the + // retry-guard edge above and the `run_task` disposition matrix in + // `harness::brain::tests` (cancel / pause / redirect all take effect). + + /// Empty twice → a graceful, non-error reply (chat never shows "Couldn't + /// send" for a transient hiccup), still two attempts. + #[tokio::test] + async fn turn_wrapper_empty_twice_is_graceful() { + let (agent, _deps) = scripted_agent(vec![Ok(String::new()), Ok(String::new())]); + let (outcome, usages) = agent.run("hi").await.expect("graceful, not an Err"); + assert!( + outcome + .reply + .to_lowercase() + .contains("temporary model hiccup"), + "got {:?}", + outcome.reply + ); + assert_eq!(usages.len(), 2); + } + + /// The Empty-vs-Hard split: only the transient empty-response class is + /// retried/softened; every other error is `Hard` and propagates loudly (no + /// blanket swallow). Driven at the classifier so it's deterministic — the + /// live agent internally retries provider errors, which would make a scripted + /// "hard error" non-deterministic. + #[test] + fn transient_empty_response_is_recognised_but_hard_errors_are_not() { + let empty = anyhow::anyhow!("The model returned an empty response. Please try again."); + assert!( + is_transient_empty_response(&empty), + "empty-response is transient" + ); + + let hard = anyhow::anyhow!("daily budget exceeded for agent 'ceo'"); + assert!( + !is_transient_empty_response(&hard), + "a budget error is NOT the transient empty class — it must propagate" + ); + } + + // --- MCP-freshness ------------------------------------------------------ + + /// In-memory secret store so `ensure` can re-resolve the runtime MCP index. + #[derive(Default)] + struct MemSecrets { + map: StdMutex>, + } + + #[async_trait] + impl SecretStore for MemSecrets { + async fn get( + &self, + _c: &CompanyId, + key: &str, + ) -> crate::Result> { + Ok(self + .map + .lock() + .unwrap() + .get(key) + .map(|v| crate::ports::types::SecretValue(v.clone()))) + } + async fn set( + &self, + _c: &CompanyId, + key: &str, + value: crate::ports::types::SecretValue, + ) -> crate::Result<()> { + self.map.lock().unwrap().insert(key.to_string(), value.0); + Ok(()) + } + } + + /// A console-added MCP server reaches the agent on the NEXT `ensure`, with no + /// restart — the roster rebuilds because the effective set, re-resolved from + /// the LIVE secret store (not the boot snapshot), changed its fingerprint. + /// This is the Parallel-Search / BrowserBase freshness bug proven end-to-end, + /// and the CI guard for issue #566: the effective-MCP fingerprint is a *term* + /// of [`HarnessPool::ensure`]'s staleness check. Both directions are pinned — + /// an unchanged set holds the fingerprint (no needless rebuild), an MCP-only + /// change moves it (rebuilt in place, without a restart). A refactor that + /// drops the term makes the post-change `ensure` early-return without storing + /// the new fingerprint: the value stops moving across the mutation and the + /// `assert_ne!` fails, rather than the restart requirement quietly returning. + #[tokio::test] + async fn ensure_rebuilds_when_a_runtime_mcp_server_is_added() { + let secrets: Arc = Arc::new(MemSecrets::default()); + let dir = tempfile::tempdir().unwrap(); + let deps = HarnessDeps { + ledgers: None, + ledger_registry: Default::default(), + provider: Arc::new(MockProvider::new("mock: ")), + provider_slug: "mock".to_string(), + serves: None, + context: Arc::new(MockContext::default()), + store: Arc::new(RecordingStore::default()), + meter: None, + workspace_root: dir.path().to_path_buf(), + workspace_git_enabled: false, + audit_root: dir.path().to_path_buf(), + model_override: None, + tasks: None, + artifacts: None, + skills: None, + skills_source_dir: None, + skills_registry: std::sync::Arc::from([]), + default_mcp_servers: Vec::new(), + mcp_servers: Vec::new(), + facts: None, + events: None, + delegations: DelegationQueue::default(), + workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), + mcp_failures: McpFailureQueue::default(), + pending_publishes: crate::harness::publish::PendingPublishQueue::default(), + workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), + run_outputs: crate::harness::orchestrator::RunOutputCache::default(), + run_output_store: None, + workflow_revisions: None, + approval_requests: ApprovalRequestQueue::default(), + secrets: Some(secrets.clone()), + web_allowed_domains: Vec::new(), + capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, + workflow_source_dir: None, + plan: None, + media: None, + composio: None, + #[cfg(feature = "chargebee")] + chargebee: None, + #[cfg(feature = "paypal")] + paypal: None, + hosting: None, + steer: crate::company::steer::InflightRegistry::default(), + run_supervisor: crate::runtime::RunSupervisor::default(), + delivery: None, + search: None, + workspace: None, + repos: None, + repo_bindings: Vec::new(), + checkouts: crate::harness::repo::CheckoutLedger::default(), + }; + let pool = HarnessPool::new(); + let rec = record(); + + pool.ensure(&rec, &deps).await.expect("first ensure"); + let before = pool + .mcp_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + + // Stability direction: with no axis changed, a redundant `ensure` is a + // no-op — the gate reuses the cached roster and the fingerprint holds, so + // the change-direction assertion below can't pass by coincidence. + pool.ensure(&rec, &deps).await.expect("redundant ensure"); + assert_eq!( + pool.mcp_fingerprint_of(&rec.id).await, + Some(before), + "an unchanged MCP set must not move the fingerprint" + ); + + // Console-add a runtime MCP server directly into the live secret store. + crate::company::mcp::save_runtime_index( + &rec.id, + secrets.as_ref(), + &[crate::company::McpServer { + name: "browserbase".into(), + endpoint: "https://api.browserbase.com/mcp".into(), + description: None, + command: None, + allowed_tools: Vec::new(), + disallowed_tools: Vec::new(), + timeout_secs: 30, + enabled: true, + auth_secret: None, + }], + ) + .await + .unwrap(); + + // Change direction: the next ensure re-resolves from the live store → + // fingerprint changes → roster rebuilt, so the new server reaches the + // agent without a restart. + pool.ensure(&rec, &deps).await.expect("post-add ensure"); + let after = pool + .mcp_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert_ne!( + before, after, + "an MCP-only change must move the staleness fingerprint (issue #566)" + ); + assert_eq!( + pool.resident_companies().await, + 1, + "same company, rebuilt in place — not a new residency" + ); + + // Stability after the change too: a further ensure with no new change is + // a no-op and the fingerprint holds at its post-change value. + pool.ensure(&rec, &deps).await.expect("final no-op ensure"); + assert_eq!(pool.mcp_fingerprint_of(&rec.id).await, Some(after)); + } + + // --- Bound-repository freshness (issue #245) ---------------------------- + + /// A bind, a credential **rotation** and a revoke each rebuild the roster on + /// the company's next turn, with no restart. + /// + /// All three are asserted because they fail differently, and the middle one + /// is the reason the fingerprint is over `(key, token_fingerprint, + /// branches)` rather than over the set of keys. A rotation changes nothing + /// about *which* repositories exist; a revoke blanks a credential while the + /// key survives for the moment before the entry is dropped. A roster keyed + /// on the key set alone holds through both, and an agent is left holding a + /// tool over a binding that can no longer fetch. + /// + /// The index is written straight into the live secret store rather than + /// through `bind`, because what is under test is the *staleness gate*, and + /// binding for real would drag a `git` fixture and a network-shaped code + /// path into a test about a hash. + #[tokio::test] + async fn ensure_rebuilds_when_a_repository_is_bound_rotated_or_revoked() { + use crate::runtime::repo_manager::types::RepoBinding; + + let secrets: Arc = Arc::new(MemSecrets::default()); + let dir = tempfile::tempdir().unwrap(); + let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); + deps.secrets = Some(secrets.clone()); + deps.repos = Some(Arc::new(crate::runtime::RepoManager::new( + CompanyId::new("acme"), + dir.path().join("repos"), + secrets.clone(), + ))); + + // The grant is what opens this axis at all: a company that does not + // explicitly grant `repo` never reads the index, so its fingerprint can + // never move. That is the fast path every other company stays on. + let mut rec = record(); + rec.manifest.tools.allow = vec!["repo".to_string()]; + + let pool = HarnessPool::new(); + let write_index = |bindings: Vec| { + let secrets = secrets.clone(); + async move { + let json = serde_json::to_string(&serde_json::json!({ "bindings": bindings })) + .expect("index json"); + secrets + .set( + &CompanyId::new("acme"), + crate::runtime::repo_manager::REPO_INDEX_KEY, + crate::ports::types::SecretValue(json), + ) + .await + .expect("write index"); + } + }; + let binding = |fingerprint: &str| RepoBinding { + key: "acme-widgets-000000000000".to_string(), + url: "https://github.com/acme/widgets".to_string(), + owner: "acme".to_string(), + repo: "widgets".to_string(), + branches: vec!["main".to_string()], + token_fingerprint: fingerprint.to_string(), + last_fetched_millis: None, + size_bytes: 0, + bound_at_millis: 1, + can_push: None, + }; + + pool.ensure(&rec, &deps).await.expect("first ensure"); + let empty = pool + .repo_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + + // Stability first, so every change assertion below cannot pass by + // coincidence. + pool.ensure(&rec, &deps).await.expect("redundant ensure"); + assert_eq!( + pool.repo_fingerprint_of(&rec.id).await, + Some(empty), + "an unchanged binding set must not move the fingerprint" + ); + + // Bind. + write_index(vec![binding("0f1e2d3c4b5a")]).await; + pool.ensure(&rec, &deps).await.expect("post-bind ensure"); + let bound = pool + .repo_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + assert_ne!(empty, bound, "a bind must move the staleness fingerprint"); + + // Rotate: same repository, same branches, new credential. + write_index(vec![binding("aaaaaaaaaaaa")]).await; + pool.ensure(&rec, &deps).await.expect("post-rotate ensure"); + let rotated = pool + .repo_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + assert_ne!( + bound, rotated, + "a credential rotation must move the fingerprint even though the \ + repository set is identical" + ); + + // Revoke. + write_index(Vec::new()).await; + pool.ensure(&rec, &deps).await.expect("post-revoke ensure"); + let revoked = pool + .repo_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + assert_ne!(rotated, revoked, "a revoke must move the fingerprint"); + assert_eq!(revoked, empty, "and must land back on the empty set"); + assert_eq!( + pool.resident_companies().await, + 1, + "same company, rebuilt in place — not a new residency" + ); + } + + /// A company that does not explicitly grant `repo` never reads the binding + /// index, so this axis is inert for it — the fast path every company that + /// does not use the feature stays on. + #[tokio::test] + async fn a_company_without_the_repo_grant_never_moves_on_this_axis() { + use crate::runtime::repo_manager::types::RepoBinding; + + let secrets: Arc = Arc::new(MemSecrets::default()); + let dir = tempfile::tempdir().unwrap(); + let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); + deps.secrets = Some(secrets.clone()); + deps.repos = Some(Arc::new(crate::runtime::RepoManager::new( + CompanyId::new("acme"), + dir.path().join("repos"), + secrets.clone(), + ))); + + // A wildcard, deliberately: `*` does not confer `repo`, so even a + // broadly-permissioned company stays off this axis. + let mut rec = record(); + rec.manifest.tools.allow = vec!["*".to_string()]; + + let pool = HarnessPool::new(); + pool.ensure(&rec, &deps).await.expect("first ensure"); + let before = pool + .repo_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + + let json = serde_json::to_string(&serde_json::json!({ + "bindings": [RepoBinding { + key: "acme-widgets-000000000000".to_string(), + url: "https://github.com/acme/widgets".to_string(), + owner: "acme".to_string(), + repo: "widgets".to_string(), + branches: vec!["main".to_string()], + token_fingerprint: "0f1e2d3c4b5a".to_string(), + last_fetched_millis: None, + size_bytes: 0, + bound_at_millis: 1, + can_push: None, + }] + })) + .unwrap(); + secrets + .set( + &CompanyId::new("acme"), + crate::runtime::repo_manager::REPO_INDEX_KEY, + crate::ports::types::SecretValue(json), + ) + .await + .unwrap(); + + pool.ensure(&rec, &deps).await.expect("post-bind ensure"); + assert_eq!( + pool.repo_fingerprint_of(&rec.id).await, + Some(before), + "an ungranted company must not read the index, let alone rebuild on it" + ); + } + + // --- Billing-credential freshness (issues #788, #789) ------------------- + + /// Saving or rotating a key in Settings → Billing must reach the agent on + /// its next turn. + /// + /// The fingerprint is the observable that makes "no restart" testable: a + /// credential that fails to move it leaves the roster cached, and the agent + /// keeps authenticating with the old key — or holds no billing tools at all + /// — until the process restarts. That failure is invisible from the tool + /// list alone, which is why this asserts the fingerprint directly. + #[tokio::test] + #[cfg(feature = "chargebee")] + async fn ensure_rebuilds_when_a_chargebee_credential_is_saved_or_rotated() { + use crate::chargebee::types::{API_KEY_SECRET, SITE_SECRET}; + + let secrets: Arc = Arc::new(MemSecrets::default()); + let dir = tempfile::tempdir().unwrap(); + let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); + deps.secrets = Some(secrets.clone()); + + // The explicit grant is what opens this axis. A `*` wildcard does not + // confer it — see the module docs. + let mut rec = record(); + rec.manifest.tools.allow = vec!["chargebee".to_string()]; + + let write = |key: &'static str, value: &'static str| { + let secrets = secrets.clone(); + async move { + secrets + .set( + &CompanyId::new("acme"), + key, + crate::ports::types::SecretValue(value.to_string()), + ) + .await + .expect("write secret"); + } + }; + + let pool = HarnessPool::new(); + pool.ensure(&rec, &deps).await.expect("first ensure"); + let unset = pool + .billing_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + + // Stability first, so every change assertion below cannot pass by + // coincidence. + pool.ensure(&rec, &deps).await.expect("redundant ensure"); + assert_eq!( + pool.billing_fingerprint_of(&rec.id).await, + Some(unset), + "an unchanged credential must not move the fingerprint" + ); + + // Half a credential is not a connection, so it must not move either — + // the pair is meaningless apart. + write(SITE_SECRET, "acme-test").await; + pool.ensure(&rec, &deps).await.expect("half ensure"); + assert_eq!( + pool.billing_fingerprint_of(&rec.id).await, + Some(unset), + "a site with no key is still no connection" + ); + + // Connect. + write(API_KEY_SECRET, "cb_first").await; + pool.ensure(&rec, &deps).await.expect("post-connect ensure"); + let connected = pool + .billing_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + assert_ne!(unset, connected, "saving a credential must rebuild"); + + // Rotate: same site, new key. This is the one a fingerprint over the + // site alone would miss, leaving the agent on the revoked key. + write(API_KEY_SECRET, "cb_rotated").await; + pool.ensure(&rec, &deps).await.expect("post-rotate ensure"); + let rotated = pool + .billing_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + assert_ne!( + connected, rotated, + "a rotation must rebuild even though the site is identical" + ); + + // Disconnect. + write(API_KEY_SECRET, "").await; + pool.ensure(&rec, &deps).await.expect("post-clear ensure"); + assert_eq!( + pool.billing_fingerprint_of(&rec.id).await, + Some(unset), + "clearing the key must land back on the unconnected fingerprint" + ); + assert_eq!( + pool.resident_companies().await, + 1, + "same company, rebuilt in place — not a new residency" + ); + } + + /// A company that does not explicitly grant `chargebee` never reads the + /// billing secrets, so this axis is inert for it — and a credential sitting + /// in its store confers nothing. Fail closed, as the module docs promise. + #[tokio::test] + #[cfg(feature = "chargebee")] + async fn a_company_without_the_chargebee_grant_never_moves_on_this_axis() { + use crate::chargebee::types::{API_KEY_SECRET, SITE_SECRET}; + + let secrets: Arc = Arc::new(MemSecrets::default()); + let dir = tempfile::tempdir().unwrap(); + let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); + deps.secrets = Some(secrets.clone()); + + // A wildcard, deliberately: it must NOT confer billing. + let mut rec = record(); + rec.manifest.tools.allow = vec!["*".to_string()]; + + let pool = HarnessPool::new(); + pool.ensure(&rec, &deps).await.expect("first ensure"); + let before = pool + .billing_fingerprint_of(&rec.id) + .await + .expect("fingerprint"); + + for (key, value) in [(SITE_SECRET, "acme-test"), (API_KEY_SECRET, "cb_key")] { + secrets + .set( + &CompanyId::new("acme"), + key, + crate::ports::types::SecretValue(value.to_string()), + ) + .await + .expect("write secret"); + } + + pool.ensure(&rec, &deps).await.expect("post-write ensure"); + assert_eq!( + pool.billing_fingerprint_of(&rec.id).await, + Some(before), + "an ungranted company must not read the billing secrets, let alone rebuild on them" + ); + } + + // --- Skill-delta freshness (issue #41) ---------------------------------- + + /// An in-memory `SkillStateStore` whose delta set a test can mutate between + /// two `ensure` calls — the same way the console Skills tab authors, edits, + /// enables, or disables a skill — so the freshness gate can be observed + /// reacting with no restart. + #[derive(Default)] + struct MemSkills { + deltas: StdMutex>, + } + + #[async_trait] + impl SkillStateStore for MemSkills { + async fn list(&self, _company: &CompanyId) -> crate::Result> { + Ok(self.deltas.lock().unwrap().clone()) + } + async fn set(&self, _company: &CompanyId, state: &SkillState) -> crate::Result<()> { + let mut deltas = self.deltas.lock().unwrap(); + match deltas.iter_mut().find(|s| s.slug == state.slug) { + Some(slot) => *slot = state.clone(), + None => deltas.push(state.clone()), + } + Ok(()) + } + async fn remove(&self, _company: &CompanyId, slug: &str) -> crate::Result { + let mut deltas = self.deltas.lock().unwrap(); + let before = deltas.len(); + deltas.retain(|s| s.slug != slug); + Ok(deltas.len() != before) + } + } + + /// A valid custom-skill delta (its `custom_doc` parses, so `materialize` + /// writes it to the scratch tree). + fn custom_skill(slug: &str, enabled: bool, body: &str) -> SkillState { + SkillState { + slug: slug.to_string(), + enabled, + source: crate::ports::skills_state::SkillSource::Custom, + custom_doc: Some(body.to_string()), + } + } + + const STANDUP_MD: &str = + "---\nname: Standup Digest\ndescription: Summarize the standup\n---\n\n# Standup Digest\n"; + + /// The scratch path a materialized skill lands at for the first roster agent + /// (`ceo`) under a company's workspace root. + fn skill_scratch(ws: &std::path::Path, slug: &str) -> std::path::PathBuf { + ws.join("acme") + .join("ceo") + .join("skill-catalog") + .join("skills") + .join(slug) + .join("SKILL.md") + } + + /// The regression: a skill authored in the console after the first roster + /// build reaches the agent on the NEXT `ensure` — the fingerprint changes, + /// the roster rebuilds in place, and the skill's `SKILL.md` materializes — + /// even though MCP / overlay / capability / composio are all unchanged. + #[tokio::test] + async fn ensure_rebuilds_when_a_custom_skill_is_authored() { + let skills = Arc::new(MemSkills::default()); + let mut fx = fixture(); + fx.deps.skills = Some(skills.clone()); + let ws = fx._dir.path().to_path_buf(); + let pool = HarnessPool::new(); + let rec = record(); + + pool.ensure(&rec, &fx.deps).await.expect("first ensure"); + let before = pool + .skill_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert!( + !skill_scratch(&ws, "standup-digest").exists(), + "no skill authored yet" + ); + + // Author a custom skill in the "console" (the live store) — no restart. + skills + .set(&rec.id, &custom_skill("standup-digest", true, STANDUP_MD)) + .await + .unwrap(); + + pool.ensure(&rec, &fx.deps).await.expect("second ensure"); + let after = pool + .skill_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert_ne!( + before, after, + "authoring a skill must change the fingerprint" + ); + assert_eq!( + pool.resident_companies().await, + 1, + "same company, rebuilt in place" + ); + assert!( + skill_scratch(&ws, "standup-digest").is_file(), + "the authored skill must surface to the agent with no restart" + ); + + // A third ensure with no change is a no-op (fingerprint stable). + pool.ensure(&rec, &fx.deps).await.expect("third ensure"); + assert_eq!(pool.skill_fingerprint_of(&rec.id).await, Some(after)); + } + + /// An unchanged delta set across two `ensure` calls keeps the fingerprint + /// stable and reuses the cached roster (the common fast path). + #[tokio::test] + async fn ensure_skill_fast_path_is_stable() { + let skills = Arc::new(MemSkills::default()); + let rec = record(); + skills + .set(&rec.id, &custom_skill("standup-digest", true, STANDUP_MD)) + .await + .unwrap(); + let mut fx = fixture(); + fx.deps.skills = Some(skills.clone()); + let pool = HarnessPool::new(); + + pool.ensure(&rec, &fx.deps).await.expect("first ensure"); + let first = pool.skill_fingerprint_of(&rec.id).await.unwrap(); + pool.ensure(&rec, &fx.deps).await.expect("second ensure"); + let second = pool.skill_fingerprint_of(&rec.id).await.unwrap(); + assert_eq!( + first, second, + "unchanged deltas keep the fingerprint stable" + ); + assert_eq!( + pool.resident_companies().await, + 1, + "roster reused, not grown" + ); + } + + /// Disabling a skill in the console drops it from the rebuilt scratch tree + /// on the next `ensure` (fingerprint moves, `SKILL.md` gone). + #[tokio::test] + async fn ensure_rebuilds_when_a_skill_is_disabled() { + let skills = Arc::new(MemSkills::default()); + let rec = record(); + skills + .set(&rec.id, &custom_skill("standup-digest", true, STANDUP_MD)) + .await + .unwrap(); + let mut fx = fixture(); + fx.deps.skills = Some(skills.clone()); + let ws = fx._dir.path().to_path_buf(); + let pool = HarnessPool::new(); + + pool.ensure(&rec, &fx.deps).await.expect("first ensure"); + let enabled_fp = pool.skill_fingerprint_of(&rec.id).await.unwrap(); + let path = skill_scratch(&ws, "standup-digest"); + assert!(path.is_file(), "an enabled skill materializes"); + + // Disable it in the console. + skills + .set(&rec.id, &custom_skill("standup-digest", false, STANDUP_MD)) + .await + .unwrap(); + pool.ensure(&rec, &fx.deps).await.expect("second ensure"); + let disabled_fp = pool.skill_fingerprint_of(&rec.id).await.unwrap(); + assert_ne!(enabled_fp, disabled_fp, "disabling changes the fingerprint"); + assert!( + !path.exists(), + "a disabled skill is dropped from the rebuilt scratch tree" + ); + assert_eq!(pool.resident_companies().await, 1, "rebuilt in place"); + } + + /// The fingerprint is order-agnostic (the store gives no ordering contract) + /// but content-sensitive (an edited `custom_doc` must trigger a rebuild). + #[test] + fn skill_delta_fingerprint_is_order_agnostic_but_content_sensitive() { + let a = custom_skill("alpha", true, "---\nname: A\ndescription: a\n---\n"); + let b = custom_skill("beta", true, "---\nname: B\ndescription: b\n---\n"); + assert_eq!( + skill_delta_fingerprint(&[a.clone(), b.clone()]), + skill_delta_fingerprint(&[b, a.clone()]), + "row order must not change the fingerprint" + ); + + let a_edited = custom_skill("alpha", true, "---\nname: A\ndescription: EDITED\n---\n"); + assert_ne!( + skill_delta_fingerprint(&[a]), + skill_delta_fingerprint(&[a_edited]), + "an edited custom_doc must change the fingerprint" + ); + } + + // --- Overlay-agent freshness (issue #71) -------------------------------- + + /// A `CompanyStore` backed by a live, mutable record — so a test can mutate + /// it between two `ensure` calls the same way the console `POST .../team` + /// route or the orchestrator's `add_agent` tool would, and observe the + /// freshness gate react. + #[derive(Default)] + struct LiveStore { + record: StdMutex>, + } + + #[async_trait] + impl CompanyStore for LiveStore { + async fn load(&self, _id: &CompanyId) -> crate::Result> { + Ok(self.record.lock().unwrap().clone()) + } + async fn save(&self, record: &CompanyRecord) -> crate::Result<()> { + *self.record.lock().unwrap() = Some(record.clone()); + Ok(()) + } + async fn list(&self) -> crate::Result> { + Ok(Vec::new()) + } + async fn append_ledger(&self, _id: &CompanyId, _entry: LedgerEntry) -> crate::Result<()> { + Ok(()) + } + } + + /// An overlay teammate added through the live company store (the same path + /// the console `POST .../team` route and the orchestrator's `add_agent` tool + /// both write through) reaches the roster on the company's NEXT `ensure` — + /// no restart — mirroring `ensure_rebuilds_when_a_runtime_mcp_server_is_added`. + #[tokio::test] + async fn ensure_rebuilds_when_an_overlay_agent_is_added() { + let live_store = Arc::new(LiveStore::default()); + let rec = record(); + live_store.save(&rec).await.unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let deps = HarnessDeps { + ledgers: None, + ledger_registry: Default::default(), + provider: Arc::new(MockProvider::new("mock: ")), + provider_slug: "mock".to_string(), + serves: None, + context: Arc::new(MockContext::default()), + store: live_store.clone(), + meter: None, + workspace_root: dir.path().to_path_buf(), + workspace_git_enabled: false, + audit_root: dir.path().to_path_buf(), + model_override: None, + tasks: None, + artifacts: None, + skills: None, + skills_source_dir: None, + skills_registry: std::sync::Arc::from([]), + default_mcp_servers: Vec::new(), + mcp_servers: Vec::new(), + facts: None, + events: None, + delegations: DelegationQueue::default(), + workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), + mcp_failures: McpFailureQueue::default(), + pending_publishes: crate::harness::publish::PendingPublishQueue::default(), + workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), + run_outputs: crate::harness::orchestrator::RunOutputCache::default(), + run_output_store: None, + workflow_revisions: None, + approval_requests: ApprovalRequestQueue::default(), + secrets: None, + web_allowed_domains: Vec::new(), + capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, + workflow_source_dir: None, + plan: None, + media: None, + composio: None, + #[cfg(feature = "chargebee")] + chargebee: None, + #[cfg(feature = "paypal")] + paypal: None, + hosting: None, + steer: crate::company::steer::InflightRegistry::default(), + run_supervisor: crate::runtime::RunSupervisor::default(), + delivery: None, + search: None, + workspace: None, + repos: None, + repo_bindings: Vec::new(), + checkouts: crate::harness::repo::CheckoutLedger::default(), + }; + let pool = HarnessPool::new(); + + pool.ensure(&rec, &deps).await.expect("first ensure"); + let before = pool + .overlay_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert_eq!(pool.resident_companies().await, 1); + // The roster is not addressable under "growth" yet. + assert!( + pool.run(&rec.id, "growth", "hi", &deps, None) + .await + .is_err(), + "the overlay teammate must not exist before it is added" + ); + + // Add a teammate directly through the live store — the same write path + // `AddAgentTool` and the console `POST .../team` route both use. + let mut updated = rec.clone(); + updated.overlay_agents.push(OverlayAgent { + id: "growth".into(), + name: "Jamie".into(), + role: "Growth Lead".into(), + description: None, + tools: Vec::new(), + }); + live_store.save(&updated).await.unwrap(); + + // Next ensure re-resolves the live store → fingerprint changes → roster + // rebuilt, so the new teammate reaches the company without a restart. + pool.ensure(&rec, &deps).await.expect("second ensure"); + let after = pool + .overlay_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert_ne!( + before, after, + "adding a teammate must change the overlay fingerprint" + ); + assert_eq!( + pool.resident_companies().await, + 1, + "same company, rebuilt in place" + ); + + let reply = pool + .run(&rec.id, "growth", "hello-marker", &deps, None) + .await + .expect("the new teammate is addressable on the very next turn") + .reply; + assert!(reply.contains("hello-marker"), "got {reply:?}"); + + // A third ensure with no further change is a no-op (fingerprint stable). + pool.ensure(&rec, &deps).await.expect("third ensure"); + assert_eq!(pool.overlay_fingerprint_of(&rec.id).await, Some(after)); + } + + // --- Capability-budget freshness (issue #108) --------------------------- + + /// A manifest that grants every tool namespace, so the roster actually builds + /// the exec tools the capability filter then trims. (The default `manifest()` + /// grants nothing, so no exec tools would be present to gate.) + fn granting_manifest() -> CompanyManifest { + toml::from_str( + r#" +[company] +name = "Acme" + +[policy] +mode = "full" + +[tools] +allow = ["shell", "code", "web", "files"] + +[[agent]] +id = "ceo" +role = "Chief Executive" +description = "Sets direction." +"#, + ) + .expect("valid manifest") + } + + fn granting_record() -> CompanyRecord { + CompanyRecord { + id: CompanyId::new("acme"), + manifest: granting_manifest(), + ledger: Vec::new(), + lifecycle: "running".to_string(), + setup: None, + overlay_agents: Vec::new(), + overlay_desk_members: Vec::new(), + overlay_desk_order: Vec::new(), + overlay_desks: Vec::new(), + overlay_workflows: Vec::new(), + overlay_budgets: Vec::new(), + overlay_policy: None, + overlay_desk_tools: Default::default(), + disabled_workflows: Vec::new(), + template_provenance: None, + } + } + + /// The `ceo` roster agent's live tool names (test introspection via the + /// public `Agent::tools()` accessor). + async fn ceo_tool_names(pool: &HarnessPool, id: &CompanyId) -> Vec { + let guard = pool.agents.read().await; + let roster = guard.get(id).expect("roster present"); + let ceo = roster + .iter() + .find(|a| a.agent_id == "ceo") + .expect("ceo present"); + let agent = ceo.agent.lock().await; + agent.tools().iter().map(|t| t.name().to_string()).collect() + } + + /// End-to-end capability gating: a plan budgeting `shell` at 100 tokens grants + /// the shell tools while spend is under budget; once a recorded turn pushes + /// period spend past the threshold, the very next `ensure` rebuilds the roster + /// with the shell namespace dropped — while intrinsic tools (memory) and the + /// ungated `files` namespace survive. Mirrors the MCP-freshness test shape. + #[tokio::test] + async fn ensure_gates_shell_tools_once_the_token_budget_is_crossed() { + let dir = tempfile::tempdir().unwrap(); + let meter = Arc::new(RecordingMeter::default()); + let plan = crate::harness::capability_budget::CapabilityPlan { + period: crate::harness::capability_budget::BudgetPeriod::Daily, + budgets: std::collections::BTreeMap::from([("shell".to_string(), 100u64)]), + total_budget: None, + }; + let deps = HarnessDeps { + ledgers: None, + ledger_registry: Default::default(), + provider: Arc::new(MockProvider::new("mock: ")), + provider_slug: "mock".to_string(), + serves: None, + context: Arc::new(MockContext::default()), + store: Arc::new(RecordingStore::default()), + meter: Some(meter.clone()), + workspace_root: dir.path().to_path_buf(), + workspace_git_enabled: false, + audit_root: dir.path().to_path_buf(), + model_override: None, + tasks: None, + artifacts: None, + skills: None, + skills_source_dir: None, + skills_registry: std::sync::Arc::from([]), + default_mcp_servers: Vec::new(), + mcp_servers: Vec::new(), + facts: None, + events: None, + delegations: DelegationQueue::default(), + workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), + mcp_failures: McpFailureQueue::default(), + pending_publishes: crate::harness::publish::PendingPublishQueue::default(), + workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), + run_outputs: crate::harness::orchestrator::RunOutputCache::default(), + run_output_store: None, + workflow_revisions: None, + approval_requests: ApprovalRequestQueue::default(), + secrets: None, + web_allowed_domains: Vec::new(), + capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, + workflow_source_dir: None, + plan: Some(plan), + media: None, + composio: None, + #[cfg(feature = "chargebee")] + chargebee: None, + #[cfg(feature = "paypal")] + paypal: None, + hosting: None, + steer: crate::company::steer::InflightRegistry::default(), + run_supervisor: crate::runtime::RunSupervisor::default(), + delivery: None, + search: None, + workspace: None, + repos: None, + repo_bindings: Vec::new(), + checkouts: crate::harness::repo::CheckoutLedger::default(), + }; + let pool = HarnessPool::new(); + let rec = granting_record(); + + // First ensure: 0 spend < 100 → shell granted. + pool.ensure(&rec, &deps).await.expect("first ensure"); + let before_fp = pool + .capability_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + let before = ceo_tool_names(&pool, &rec.id).await; + assert!(before.contains(&"shell".to_string()), "got {before:?}"); + assert!( + before.contains(&"read_workspace_state".to_string()), + "got {before:?}" + ); + // `memory_store`/`memory_recall` are currently withheld altogether + // (see `harness::build::memory_tools`'s doc comment) — openhuman + // removed the constructor seam that let either tool act on a + // company's own `ContextStore` rather than one shared, + // unconfigured store. `file_read` is this test's example of an + // intrinsic, ungated tool instead. + assert!( + before.contains(&"file_read".to_string()), + "ungated files namespace must be present: {before:?}" + ); + + // Record a turn that burns 150 inference tokens — past the 100 budget. + meter + .record( + &rec.id, + &UsageSample { + at_millis: crate::ports::now_millis(), + agent: "ceo".into(), + provider: "managed".into(), + input_tokens: 100, + output_tokens: 50, + cached_input_tokens: 0, + cost_usd: 0.0, + kind: crate::ports::SampleKind::Inference, + run_id: None, + }, + ) + .await + .unwrap(); + + // Second ensure: 150 >= 100 → shell exhausted → roster rebuilt without it. + pool.ensure(&rec, &deps).await.expect("second ensure"); + let after_fp = pool + .capability_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert_ne!( + before_fp, after_fp, + "crossing the budget must change the capability fingerprint" + ); + assert_eq!(pool.resident_companies().await, 1, "rebuilt in place"); + + let after = ceo_tool_names(&pool, &rec.id).await; + assert!( + !after.contains(&"shell".to_string()), + "shell must be gated off once exhausted: {after:?}" + ); + assert!( + !after.contains(&"read_workspace_state".to_string()), + "the whole shell namespace drops: {after:?}" + ); + assert!( + after.contains(&"file_read".to_string()), + "ungated files namespace survives gating: {after:?}" + ); + + // Third ensure with no new spend → no rebuild (fingerprint stable). + pool.ensure(&rec, &deps).await.expect("third ensure"); + assert_eq!( + pool.capability_fingerprint_of(&rec.id).await, + Some(after_fp) + ); + } + + /// With no plan wired, the capability fingerprint is stable across ensures — + /// gating stays off, byte-identical to Cell A (no rebuild on this axis). + #[tokio::test] + async fn ensure_without_a_plan_never_gates() { + let fx = fixture(); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &fx.deps).await.expect("first ensure"); + let fp = pool + .capability_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + pool.ensure(&rec, &fx.deps).await.expect("second ensure"); + assert_eq!( + pool.capability_fingerprint_of(&rec.id).await, + Some(fp), + "no plan → stable fingerprint → no capability-driven rebuild" + ); + } + + /// Builds a `HarnessDeps` carrying the given plan + meter, for the total- + /// ceiling dispatch tests (issue #188). Everything else is the inert fixture + /// wiring (mock provider/context, recording store). + fn deps_with_plan( + dir: &std::path::Path, + context: Arc, + meter: Option>, + plan: Option, + ) -> HarnessDeps { + HarnessDeps { + ledgers: None, + ledger_registry: Default::default(), + provider: Arc::new(MockProvider::new("mock: ")), + provider_slug: "mock".to_string(), + serves: None, + context, + store: Arc::new(RecordingStore::default()), + meter, + workspace_root: dir.to_path_buf(), + workspace_git_enabled: false, + audit_root: dir.to_path_buf(), + model_override: None, + tasks: None, + skills: None, + skills_source_dir: None, + skills_registry: std::sync::Arc::from([]), + default_mcp_servers: Vec::new(), + mcp_servers: Vec::new(), + facts: None, + events: None, + delegations: DelegationQueue::default(), + workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), + mcp_failures: McpFailureQueue::default(), + pending_publishes: crate::harness::publish::PendingPublishQueue::default(), + workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), + run_outputs: crate::harness::orchestrator::RunOutputCache::default(), + run_output_store: None, + workflow_revisions: None, + approval_requests: ApprovalRequestQueue::default(), + secrets: None, + web_allowed_domains: Vec::new(), + capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, + workflow_source_dir: None, + plan, + media: None, + composio: None, + #[cfg(feature = "chargebee")] + chargebee: None, + #[cfg(feature = "paypal")] + paypal: None, + hosting: None, + artifacts: None, + steer: crate::company::steer::InflightRegistry::default(), + run_supervisor: crate::runtime::RunSupervisor::default(), + delivery: None, + search: None, + workspace: None, + repos: None, + repo_bindings: Vec::new(), + checkouts: crate::harness::repo::CheckoutLedger::default(), + } + } + + /// The hard total-token ceiling (issue #188): once the tenant's total period + /// spend crosses the plan's `total_budget`, the very next dispatch is refused + /// **before any model call** — the reply is the fixed operator notice, the + /// prompt is never echoed (proving the model was not run), and no fabricated + /// outcome lands in memory. A turn under the ceiling still runs normally. + #[tokio::test] + async fn run_refuses_dispatch_once_the_total_ceiling_is_crossed() { + let dir = tempfile::tempdir().unwrap(); + let context = Arc::new(MockContext::default()); + let meter = Arc::new(RecordingMeter::default()); + let plan = crate::harness::capability_budget::CapabilityPlan { + period: crate::harness::capability_budget::BudgetPeriod::Daily, + budgets: std::collections::BTreeMap::new(), + total_budget: Some(100), + }; + let deps = deps_with_plan( + dir.path(), + context.clone(), + Some(meter.clone() as Arc), + Some(plan), + ); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &deps).await.expect("ensure"); + + // Under the ceiling (0 spend < 100): the turn runs and echoes the prompt. + let ok = pool + .run(&rec.id, "ceo", "hello-marker", &deps, None) + .await + .expect("under-ceiling turn runs") + .reply; + assert!( + ok.contains("hello-marker"), + "under the ceiling the model runs: {ok:?}" + ); + + // Push total period spend to 150 — past the 100-token ceiling. + meter + .record( + &rec.id, + &UsageSample { + at_millis: crate::ports::now_millis(), + agent: "ceo".into(), + provider: "managed".into(), + input_tokens: 100, + output_tokens: 50, + cached_input_tokens: 0, + cost_usd: 0.0, + kind: crate::ports::SampleKind::Inference, + run_id: None, + }, + ) + .await + .unwrap(); + + let before = context + .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) + .await + .unwrap() + .len(); + + // Over the ceiling: dispatch is refused with a benign notice — NOT an Err. + let refused = pool + .run(&rec.id, "ceo", "should-not-echo", &deps, None) + .await + .expect("a refusal is a benign outcome, not a hard error") + .reply; + assert_eq!( + refused, TOTAL_BUDGET_EXHAUSTED_NOTICE, + "the refusal returns the fixed operator notice" + ); + assert!( + !refused.contains("should-not-echo"), + "the model was never called, so the prompt is not echoed: {refused:?}" + ); + + // A refused turn writes no outcome back to memory. + let after = context + .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) + .await + .unwrap() + .len(); + assert_eq!(before, after, "a refused turn stores nothing in memory"); + } + + /// Issue #416, the reason [`HarnessPool::total_ceiling_refusal`] was + /// extracted rather than copied: a confined turn reaches nothing, but it + /// still spends model tokens, so the tenant's ceiling refuses it exactly as + /// it refuses a roster dispatch. Without this test the gate could be dropped + /// from `run_confined` and every other test would stay green — the copilot + /// would simply keep spending past the cap. + #[tokio::test] + async fn a_confined_turn_is_refused_once_the_total_ceiling_is_crossed() { + let dir = tempfile::tempdir().unwrap(); + let context = Arc::new(MockContext::default()); + let meter = Arc::new(RecordingMeter::default()); + let plan = crate::harness::capability_budget::CapabilityPlan { + period: crate::harness::capability_budget::BudgetPeriod::Daily, + budgets: std::collections::BTreeMap::new(), + total_budget: Some(100), + }; + let deps = deps_with_plan( + dir.path(), + context.clone(), + Some(meter.clone() as Arc), + Some(plan), + ); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &deps).await.expect("ensure"); + + let confinement = confine::Confinement::workflow("weekly_report"); + let thread = Some("workflow-copilot:weekly_report"); + + // Under the ceiling the copilot answers, so the refusal below is the + // ceiling talking and not the confined path failing to run at all. + let ok = pool + .run_confined(&rec.id, "Acme", "hello-marker", &deps, thread, &confinement) + .await + .expect("under-ceiling confined turn runs") + .reply; + assert!( + ok.contains("hello-marker"), + "under the ceiling the model runs: {ok:?}" + ); + + // Push total period spend past the 100-token ceiling. + meter + .record( + &rec.id, + &UsageSample { + at_millis: crate::ports::now_millis(), + agent: "ceo".into(), + provider: "managed".into(), + input_tokens: 100, + output_tokens: 50, + cached_input_tokens: 0, + cost_usd: 0.0, + kind: crate::ports::SampleKind::Inference, + run_id: None, + }, + ) + .await + .unwrap(); + + let refused = pool + .run_confined( + &rec.id, + "Acme", + "should-not-echo", + &deps, + thread, + &confinement, + ) + .await + .expect("a refusal is a benign outcome, not a hard error") + .reply; + assert_eq!( + refused, TOTAL_BUDGET_EXHAUSTED_NOTICE, + "the copilot must not keep spending past the tenant ceiling" + ); + assert!( + !refused.contains("should-not-echo"), + "the model was never called, so the prompt is not echoed: {refused:?}" + ); + } + + /// Fail-closed tradeoff (issue #188): with a total ceiling configured but no + /// meter to read spend from, the hard refusal does NOT fire — a transient + /// unreadable-spend condition must not brick every turn. The turn runs (the + /// per-namespace fail-closed roster already handles exec-tool stripping). + #[tokio::test] + async fn run_does_not_refuse_when_spend_is_unreadable() { + let dir = tempfile::tempdir().unwrap(); + let context = Arc::new(MockContext::default()); + // A zero ceiling would refuse from the first token IF spend were readable; + // with no meter wired the gate must defer, not brick. + let plan = crate::harness::capability_budget::CapabilityPlan { + period: crate::harness::capability_budget::BudgetPeriod::Daily, + budgets: std::collections::BTreeMap::new(), + total_budget: Some(0), + }; + let deps = deps_with_plan(dir.path(), context.clone(), None, Some(plan)); + let pool = HarnessPool::new(); + let rec = record(); + pool.ensure(&rec, &deps).await.expect("ensure"); + + let reply = pool + .run(&rec.id, "ceo", "hello-marker", &deps, None) + .await + .expect("no meter must not brick the turn") + .reply; + assert!( + reply.contains("hello-marker"), + "an unreadable ceiling defers to running the turn: {reply:?}" + ); + assert_ne!( + reply, TOTAL_BUDGET_EXHAUSTED_NOTICE, + "the hard refusal must not fire without a spend read" + ); + } + + // --- The per-agent daily spend cap at dispatch (issue #304) -------------- + + /// A company whose `ceo` carries a $5/day cap and whose `engineer` carries + /// none — the pair that proves the gate is per-teammate, not per-company. + fn capped_record() -> CompanyRecord { + let manifest: CompanyManifest = toml::from_str( + r#" +[company] +name = "Acme" + +[policy] +mode = "full" + +[[agent]] +id = "ceo" +role = "Chief Executive" +description = "Sets direction." +budget_usd_daily = 5.0 + +[[agent]] +id = "engineer" +role = "Engineer" +description = "Builds the product." +"#, + ) + .expect("valid manifest"); + CompanyRecord { + manifest, + ..record() + } + } + + /// A `$usd` inference sample for `agent`, stamped at `at_millis`. + fn spend_sample(agent: &str, usd: f64, at_millis: u64) -> UsageSample { + UsageSample { + at_millis, + agent: agent.into(), + provider: "managed".into(), + input_tokens: 0, + output_tokens: 0, + cached_input_tokens: 0, + cost_usd: usd, + kind: crate::ports::SampleKind::Inference, + run_id: None, + } + } + + /// The heart of #304 at the layer that carries the money: once a teammate + /// has spent its manifest `budget_usd_daily`, its next dispatch is refused + /// **before any model call** — while its uncapped colleague keeps working. + /// + /// This is the layer that matters, because the dominant spend stream is + /// inference and inference never reaches a `ToolPolicy`. Gating only priced + /// tool calls would leave a capped teammate free to burn its budget many + /// times over on model turns alone. + #[tokio::test] + async fn run_refuses_dispatch_for_a_teammate_over_its_daily_cap() { + let dir = tempfile::tempdir().unwrap(); + let context = Arc::new(MockContext::default()); + let meter = Arc::new(RecordingMeter::default()); + let rec = capped_record(); + + // The CEO has spent its whole $5 today. The engineer has spent nothing. + meter + .record( + &rec.id, + &spend_sample("ceo", 5.00, crate::ports::now_millis()), + ) + .await + .unwrap(); + + let deps = deps_with_plan( + dir.path(), + context.clone(), + Some(meter.clone() as Arc), + None, + ); + let pool = HarnessPool::new(); + pool.ensure(&rec, &deps).await.expect("ensure"); + + let samples_before = meter.samples.lock().unwrap().len(); + let memory_before = context + .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) + .await + .unwrap() + .len(); + + let refused = pool + .run(&rec.id, "ceo", "should-not-echo", &deps, None) + .await + .expect("a refusal is a benign outcome, not a hard error") + .reply; + assert_eq!( + refused, + agent_budget_exhausted_notice("ceo", 5.0), + "the refusal names the teammate, its cap and the reset" + ); + assert!( + !refused.contains("should-not-echo"), + "the model was never called, so the prompt is not echoed: {refused:?}" + ); + assert_eq!( + meter.samples.lock().unwrap().len(), + samples_before, + "a pre-model-call refusal meters nothing" + ); + assert_eq!( + context + .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) + .await + .unwrap() + .len(), + memory_before, + "a refused turn stores no fabricated outcome" + ); + + // The cap is per-teammate: the uncapped engineer is untouched, and the + // CEO's spend does not count against it. + let ok = pool + .run(&rec.id, "engineer", "hello-marker", &deps, None) + .await + .expect("an uncapped teammate keeps working") + .reply; + assert!( + ok.contains("hello-marker"), + "one teammate's exhausted budget must not stop the company: {ok:?}" + ); + } + + // --- Console budget overrides, live (issue #343) ------------------------- + + /// **The no-restart proof.** A daily cap written through the company store — + /// the exact path `PUT …/team/{id}/budget` writes through — is enforced on + /// the company's **next dispatch**, in one process, with no restart and no + /// redeploy. + /// + /// This is the whole of #343 at the layer that decides whether a teammate + /// works. Before it, `budget_usd_daily` was readable only from the manifest, + /// which is a boot snapshot baked into the tenant image — so an operator + /// whose teammate had stopped had no remedy short of us shipping a new + /// image. The four phases walk exactly that operator's day: + /// + /// A. the CEO has spent its manifest $5 and is refused (issue #304, and + /// the state that motivates the issue); + /// B. an admin **raises** the cap to $50 — the stopped teammate works + /// again on its very next turn. This is the acceptance criterion; + /// C. the admin sets the cap to **$0** — a real cap of nothing, refused + /// from the first cent; + /// D. the admin **clears** the cap — an explicitly-uncapped override that + /// beats the manifest's $5 even with $5 already spent, so the teammate + /// works again. + /// + /// C and D are the same route with different bodies and they must not + /// resolve alike: C refuses, D runs. That is "clearing is distinct from + /// zeroing" asserted on live behaviour rather than on a type. + /// + /// Throughout, the pool holds **one** resident company and is never + /// reconstructed — `resident_companies()` stays 1 and the same `pool` binding + /// serves every phase — so the only mechanism that can be carrying these + /// changes is the budget fingerprint flipping and `ensure` rebuilding the + /// roster in place. Each phase asserts that fingerprint actually moved. + #[tokio::test] + async fn a_budget_written_through_the_store_is_enforced_on_the_next_dispatch() { + use crate::ports::types::{Actor, ActorKind, BudgetOverride}; + + let dir = tempfile::tempdir().unwrap(); + let context = Arc::new(MockContext::default()); + let meter = Arc::new(RecordingMeter::default()); + let rec = capped_record(); + + // A live store, so `ensure` re-resolves the overrides the way it does in + // production. `deps_with_plan`'s default store is inert. + let live_store = Arc::new(LiveStore::default()); + live_store.save(&rec).await.unwrap(); + + // The CEO has already spent its manifest $5 today. + meter + .record( + &rec.id, + &spend_sample("ceo", 5.00, crate::ports::now_millis()), + ) + .await + .unwrap(); + + let mut deps = deps_with_plan( + dir.path(), + context.clone(), + Some(meter.clone() as Arc), + None, + ); + deps.store = live_store.clone(); + + // ONE pool for the whole test. Nothing below reconstructs it, so nothing + // below can be smuggling in a restart. + let pool = HarnessPool::new(); + + /// Writes an override through the store exactly as the console route + /// does, and returns the record for the next `ensure`. + fn with_override(base: &CompanyRecord, cap: Option) -> CompanyRecord { + let mut next = base.clone(); + next.overlay_budgets = vec![BudgetOverride { + agent_id: "ceo".to_string(), + budget_usd_daily: cap, + set_by: Actor { + kind: ActorKind::User, + id: "user-admin".to_string(), + }, + at_millis: crate::ports::now_millis(), + }]; + next + } + + // --- A. The manifest cap is spent: the teammate is stopped. ---------- + pool.ensure(&rec, &deps).await.expect("ensure A"); + let fp_manifest = pool + .budget_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + let refused = pool + .run(&rec.id, "ceo", "should-not-echo", &deps, None) + .await + .expect("a refusal is a benign outcome") + .reply; + assert_eq!( + refused, + agent_budget_exhausted_notice("ceo", 5.0), + "phase A: the manifest's $5 cap is spent, so dispatch is refused" + ); + + // --- B. An admin raises the cap. The teammate works again. ----------- + live_store + .save(&with_override(&rec, Some(50.0))) + .await + .unwrap(); + pool.ensure(&rec, &deps).await.expect("ensure B"); + let fp_raised = pool + .budget_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert_ne!( + fp_manifest, fp_raised, + "phase B: setting a cap must move the budget fingerprint, or the \ + cached roster is reused and the change never reaches the gate" + ); + assert_eq!( + pool.resident_companies().await, + 1, + "phase B: the same company, rebuilt in place — not a new process" + ); + let unblocked = pool + .run(&rec.id, "ceo", "hello-marker", &deps, None) + .await + .expect("the raised cap unblocks the teammate") + .reply; + assert!( + unblocked.contains("hello-marker"), + "phase B: raising the cap from the console must unblock the stopped \ + teammate on its very next dispatch, with no restart: {unblocked:?}" + ); + + // --- C. The admin sets the cap to zero. Zero is a real cap. ---------- + live_store + .save(&with_override(&rec, Some(0.0))) + .await + .unwrap(); + pool.ensure(&rec, &deps).await.expect("ensure C"); + let fp_zero = pool + .budget_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert_ne!(fp_raised, fp_zero, "phase C: lowering a cap is a change"); + let zeroed = pool + .run(&rec.id, "ceo", "should-not-echo", &deps, None) + .await + .expect("a refusal is a benign outcome") + .reply; + assert_eq!( + zeroed, + agent_budget_exhausted_notice("ceo", 0.0), + "phase C: a $0 cap refuses from the first cent" + ); + + // --- D. The admin clears the cap. Cleared is not zero. --------------- + live_store.save(&with_override(&rec, None)).await.unwrap(); + pool.ensure(&rec, &deps).await.expect("ensure D"); + let fp_cleared = pool + .budget_fingerprint_of(&rec.id) + .await + .expect("fingerprinted"); + assert_ne!( + fp_zero, fp_cleared, + "phase D: 'no cap' and 'a cap of $0' must not hash alike — if they \ + did, clearing a cap would silently leave the teammate at zero" + ); + let cleared = pool + .run(&rec.id, "ceo", "hello-marker", &deps, None) + .await + .expect("an explicitly-uncapped teammate runs") + .reply; + assert!( + cleared.contains("hello-marker"), + "phase D: an explicitly-uncapped override beats the manifest's $5 \ + even with $5 already spent today: {cleared:?}" + ); + + // Nothing above restarted anything. + assert_eq!( + pool.resident_companies().await, + 1, + "one company, rebuilt in place across all four phases" + ); + + // A further `ensure` with no change is a no-op: the axis is not thrashing + // the roster (and dropping live agent sessions) on every turn. + pool.ensure(&rec, &deps).await.expect("ensure idempotent"); + assert_eq!(pool.budget_fingerprint_of(&rec.id).await, Some(fp_cleared)); + } + + /// An **overlay** teammate — one added from the console, with no manifest + /// row — can be capped through the same override, and is refused when it has + /// spent it. Before #343 an overlay teammate was unconditionally uncapped + /// ("overlay teammates are uncapped in v1"), so this is a capability that did + /// not exist rather than a behaviour that changed. + #[tokio::test] + async fn an_overlay_teammate_can_be_capped_from_the_console() { + use crate::ports::types::{Actor, ActorKind, BudgetOverride}; + + let dir = tempfile::tempdir().unwrap(); + let context = Arc::new(MockContext::default()); + let meter = Arc::new(RecordingMeter::default()); + + let mut rec = record(); + rec.overlay_agents.push(OverlayAgent { + id: "growth".into(), + name: "Jamie".into(), + role: "Growth Lead".into(), + description: None, + tools: Vec::new(), + }); + let live_store = Arc::new(LiveStore::default()); + live_store.save(&rec).await.unwrap(); + + let mut deps = deps_with_plan( + dir.path(), + context.clone(), + Some(meter.clone() as Arc), + None, + ); + deps.store = live_store.clone(); + let pool = HarnessPool::new(); + + // Uncapped to begin with: it answers. + pool.ensure(&rec, &deps).await.expect("ensure"); + let reply = pool + .run(&rec.id, "growth", "hello-marker", &deps, None) + .await + .expect("an uncapped overlay teammate answers") + .reply; + assert!(reply.contains("hello-marker"), "got {reply:?}"); + + // The operator caps it at $1 and it has already spent $2. + meter + .record( + &rec.id, + &spend_sample("growth", 2.00, crate::ports::now_millis()), + ) + .await + .unwrap(); + let mut capped = rec.clone(); + capped.overlay_budgets = vec![BudgetOverride { + agent_id: "growth".to_string(), + budget_usd_daily: Some(1.0), + set_by: Actor { + kind: ActorKind::User, + id: "user-admin".to_string(), + }, + at_millis: crate::ports::now_millis(), + }]; + live_store.save(&capped).await.unwrap(); + + pool.ensure(&rec, &deps).await.expect("ensure again"); + let refused = pool + .run(&rec.id, "growth", "should-not-echo", &deps, None) + .await + .expect("a refusal is a benign outcome") + .reply; + assert_eq!( + refused, + agent_budget_exhausted_notice("growth", 1.0), + "a console-added teammate is capped by the same gate as a manifest one" + ); + } + + /// Fail-open pin, mirroring #188's documented tradeoff exactly: with a cap + /// set but spend unreadable, the turn RUNS. + /// + /// A `$0` cap would refuse from the first cent if spend were readable, so a + /// meter that errors is the only reason this turn can proceed. Bricking a + /// teammate's cognition on a flaky read is a strictly worse failure mode + /// than one day of overspend — and unlike the policy arm's park, a turn-level + /// refusal offers the operator nothing to approve. + #[tokio::test] + async fn run_does_not_refuse_a_capped_teammate_when_spend_is_unreadable() { + let dir = tempfile::tempdir().unwrap(); + let context = Arc::new(MockContext::default()); + let manifest: CompanyManifest = toml::from_str( + r#" +[company] +name = "Acme" + +[policy] +mode = "full" + +[[agent]] +id = "ceo" +role = "Chief Executive" +description = "Sets direction." +budget_usd_daily = 0.0 +"#, + ) + .expect("valid manifest"); + let rec = CompanyRecord { + manifest, + ..record() + }; + + let deps = deps_with_plan( + dir.path(), + context.clone(), + Some(Arc::new(FailingMeter) as Arc), + None, + ); + let pool = HarnessPool::new(); + pool.ensure(&rec, &deps).await.expect("ensure"); + + let reply = pool + .run(&rec.id, "ceo", "hello-marker", &deps, None) + .await + .expect("an unreadable budget must not brick the teammate") + .reply; + assert!( + reply.contains("hello-marker"), + "an unreadable cap defers to running the turn: {reply:?}" + ); + + // ...and with no meter at all, the same deferral. + let no_meter = deps_with_plan(dir.path(), context.clone(), None, None); + let pool = HarnessPool::new(); + pool.ensure(&rec, &no_meter).await.expect("ensure"); + let reply = pool + .run(&rec.id, "ceo", "hello-marker", &no_meter, None) + .await + .expect("no meter must not brick the teammate") + .reply; + assert!(reply.contains("hello-marker"), "no meter defers: {reply:?}"); + } + + /// The cap is the UTC calendar day: yesterday's $9 does not refuse today's + /// first turn. Depends on `RecordingMeter` honouring `since_millis`. + #[tokio::test] + async fn a_yesterday_stamped_spend_does_not_refuse_todays_dispatch() { + let dir = tempfile::tempdir().unwrap(); + let context = Arc::new(MockContext::default()); + let meter = Arc::new(RecordingMeter::default()); + let rec = capped_record(); + + let yesterday = + crate::metering::utc_day_start_millis(crate::ports::now_millis()).saturating_sub(1); + meter + .record(&rec.id, &spend_sample("ceo", 9.00, yesterday)) + .await + .unwrap(); + + let deps = deps_with_plan( + dir.path(), + context.clone(), + Some(meter.clone() as Arc), + None, + ); + let pool = HarnessPool::new(); + pool.ensure(&rec, &deps).await.expect("ensure"); + + let reply = pool + .run(&rec.id, "ceo", "hello-marker", &deps, None) + .await + .expect("a new day admits the turn") + .reply; + assert!( + reply.contains("hello-marker"), + "the cap resets at 00:00Z; yesterday's spend is spent: {reply:?}" + ); + } + + // ----------------------------------------------------------------------- + // The approval gate's coverage over the live toolbelt (issue #443) + // ----------------------------------------------------------------------- + + /// Build one agent and return the tools it actually received. + /// + /// A local mirror of `build`'s own `built_tool_names` — that one is private + /// to its test module, and this file owns `deps_with_plan`, which is the + /// expensive half. + fn belt(grants: &[&str], is_orchestrator: bool, wire_everything: bool) -> Vec { + let dir = tempfile::tempdir().expect("tempdir"); + let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); + if wire_everything { + // The three tool families gated on a wired dependency rather than + // on a cargo feature. Without these the belt is missing exactly the + // tools most likely to be misclassified — the workspace writes and + // the priced search. + deps.workspace = Some(Arc::new(crate::store::FsOps::new(dir.path()))); + deps.artifacts = Some(Arc::new(crate::store::FsOps::new(dir.path()))); + deps.search = Some(crate::harness::search::SearchBackend::new( + "https://api.example.test".to_string(), + crate::company::credentials::Credential::from_value("managed-platform-token"), + crate::company::DEFAULT_SEARCH_DAILY_CALLS, + )); + // Issue #245: a repository manager AND a binding, because the tools + // are gated on both — with a manager and nothing bound the belt + // would be missing `repo_checkout` / `repo_pr` and this check would + // pass while never having looked at them, which is the exact way + // `describe_skill` stayed invisible here while parking in + // production. + // Issue #752 added a fourth gate: a backend that keeps the + // credential off this container's disk. Declared here for the same + // reason the binding below is — without it the belt would be + // missing `repo_checkout` / `repo_pr` and this check would pass + // while never having looked at them. + deps.repos = Some(Arc::new( + crate::runtime::RepoManager::new( + CompanyId::new("acme"), + dir.path().join("repos"), + Arc::new(crate::store::FsSecretStore::new(dir.path())), + ) + .with_storage_kind(crate::store::StorageKind::Mongodb), + )); + deps.repo_bindings = vec![crate::runtime::repo_manager::types::RepoBinding { + key: "acme-widgets-000000000000".to_string(), + url: "https://github.com/acme/widgets".to_string(), + owner: "acme".to_string(), + repo: "widgets".to_string(), + branches: vec!["main".to_string()], + token_fingerprint: "0f1e2d3c4b5a".to_string(), + last_fetched_millis: None, + size_bytes: 0, + bound_at_millis: 1, + can_push: None, + }]; + // A registered MCP server is what puts `mcp_list_servers`, + // `mcp_list_tools` and `mcp_call_tool` on the belt — the three + // tools issue #443 is about. Without one the coverage check would + // pass while never having looked at them. + // A skills source dir is what puts `list_skills`, `describe_skill` + // and `read_skill_resource` on the belt (named for skills since + // issue #845; upstream calls them `*_workflow*`). Leaving it `None` + // is how those three stayed invisible to this check while + // `describe_workflow` parked in production. + let company_src = dir.path().join("company-src"); + std::fs::create_dir_all(company_src.join("skills").join("brief")).expect("skill dir"); + std::fs::write( + company_src.join("skills").join("brief").join("SKILL.md"), + "---\nname: brief\ndescription: Write a brief\n---\n\nWrite one.\n", + ) + .expect("skill file"); + deps.skills_source_dir = Some(company_src); + deps.mcp_servers = vec![McpServerDecl { + name: "notes".to_string(), + endpoint: "https://mcp.example.test".to_string(), + description: None, + allowed_tools: Vec::new(), + disallowed_tools: Vec::new(), + timeout_secs: 30, + enabled: true, + source: crate::company::mcp::McpSource::Runtime, + auth: crate::company::mcp::AuthMaterial::None, + }]; + } + let manifest_agent = ManifestAgent { + global: false, + id: "desk".to_string(), + role: "Desk Lead".to_string(), + name: None, + description: None, + tier: None, + harness: None, + tools: Vec::new(), + delegates_to: Vec::new(), + context: None, + budget_usd_daily: None, + prompt: None, + prompt_files: Vec::new(), + prompt_files_resolved: Vec::new(), + classes: Vec::new(), + ledgers: None, + can_declare_ledgers: true, + }; + let policy = ApprovalPolicy::new(&Policy::default(), None); + let grants: Vec = grants.iter().map(|g| g.to_string()).collect(); + let agent = build::build_agent( + &CompanyId::new("acme"), + "Acme", + &manifest_agent, + policy, + &deps, + &grants, + &[], + &[], + is_orchestrator, + ) + .expect("agent builds"); + agent.tools().iter().map(|t| t.name().to_string()).collect() + } + + /// **The mechanism issue #443 asks for.** Every tool this crate can put in + /// front of an agent must be classified in + /// [`crate::policy::consequence`], or this fails. + /// + /// Three families had needed the same carve-out before it, each added after + /// somebody hit it, and what the gate did with the ones nobody hit was + /// silent: the tool simply started asking for permission, and whoever + /// noticed was an operator wondering why a read needed approving. That is + /// how `mcp_list_servers` — which the agent persona *instructs* every agent + /// to call — came to cost an approval, and how `file_read`, `glob` and + /// `grep` came to park with nobody reporting it. + /// + /// A tool declaring its own consequence to the gate at call time would be + /// better, and is not reachable: openhuman's `ToolPolicy` surface hands the + /// bridge a name and arguments, never the tool. So the declaration is + /// checked against the live belt here instead — the issue's own stated + /// fallback, "exhaustive by construction rather than by memory". + /// + /// Feature-aware by construction: it enumerates whatever this build wires, + /// so a family behind a cargo feature is covered by the lane that enables + /// it rather than by a `cfg` branch that has to be kept in step. + #[test] + fn every_registered_tool_is_declared() { + let declared: std::collections::BTreeSet<&str> = + crate::policy::consequence::declared_tools().collect(); + let mut live: std::collections::BTreeSet = std::collections::BTreeSet::new(); + for (grants, orchestrator, everything) in [ + (&["*"][..], false, false), + (&["*"][..], true, false), + (&["*"][..], false, true), + (&["*"][..], true, true), + ( + &["workspace", "search", "media", "composio", "repo"][..], + false, + true, + ), + ] { + live.extend(belt(grants, orchestrator, everything)); + } + // A vacuity guard with teeth. `!live.is_empty()` would not notice the + // belt quietly narrowing to the tools nobody was worried about, and + // three of the four names below are the ones the issues are about. + for expected in [ + "shell", + "workspace_write", + "file_read", + "describe_skill", + "repo_checkout", + #[cfg(feature = "mcp")] + "mcp_list_servers", + #[cfg(feature = "mcp")] + "mcp_call_tool", + ] { + assert!( + live.contains(expected), + "the belt builder stopped wiring `{expected}`, so this check has \ + narrowed without anyone deciding to narrow it: {live:?}" + ); + } + let undeclared: Vec<&String> = live + .iter() + .filter(|name| !declared.contains(name.as_str())) + .collect(); + assert!( + undeclared.is_empty(), + "these tools are wired onto a live agent but nobody has said what they can \ + reach, so the gate is guessing from their names and they cannot be granted \ + standing: {undeclared:?}. Add them to `crate::policy::consequence::DECLARED`." + ); + } + + /// The one-directional cross-check on the declaration. + /// + /// A tool's own `permission_level()` is NOT trustworthy as the authority — + /// it defaults to `ReadOnly`, and upstream tools that plainly mutate + /// (`git_operations`, `memory_store`) never override it, so believing a + /// `ReadOnly` claim would wave a write straight through the gate. But the + /// claims in the *other* direction are deliberate: nothing declares itself + /// `Execute` or `Dangerous` by accident. So those are checked, and a + /// `ReadOnly` claim is ignored. + #[test] + fn nothing_that_declares_itself_executable_is_internal_or_grantable() { + use oh::tools::traits::PermissionLevel; + let dir = tempfile::tempdir().expect("tempdir"); + let deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); + let manifest_agent = ManifestAgent { + global: false, + id: "desk".to_string(), + role: "Desk Lead".to_string(), + name: None, + description: None, + tier: None, + harness: None, + tools: Vec::new(), + delegates_to: Vec::new(), + context: None, + budget_usd_daily: None, + prompt: None, + prompt_files: Vec::new(), + prompt_files_resolved: Vec::new(), + classes: Vec::new(), + ledgers: None, + can_declare_ledgers: true, + }; + let agent = build::build_agent( + &CompanyId::new("acme"), + "Acme", + &manifest_agent, + ApprovalPolicy::new(&Policy::default(), None), + &deps, + &["*".to_string()], + &[], + &[], + true, + ) + .expect("agent builds"); + let args = serde_json::json!({}); + let mut checked = 0; + for tool in agent.tools() { + if !matches!( + tool.permission_level(), + PermissionLevel::Execute | PermissionLevel::Dangerous + ) { + continue; + } + checked += 1; + let verdict = crate::policy::consequence_of(tool.name(), &args); + assert!( + verdict.reach.denied_under_readonly(), + "`{}` declares itself executable but a read-only desk would allow it", + tool.name() + ); + assert!( + !verdict.standing.is_grantable(), + "`{}` declares itself executable and must not be grantable", + tool.name() + ); + } + assert!(checked > 0, "no executable tool was on the belt to check"); + } + + // --- Per-company billing resolution (issues #788, #789) ----------------- + // + // `resolve_chargebee` / `resolve_paypal` are what actually decide whether a + // company's agents get billing tools on a given turn — `HarnessPool::ensure` + // re-resolves them every turn, and `RuntimeBuilder::build` runs the same + // three-way decision once at boot. All three branches are silent when they + // go wrong: a dropped grant check wires tools the manifest never allowed, and + // a read error collapsed into "no credential" disconnects a working + // integration on one transient store hiccup. + + /// A secret store that reads back what was seeded, or fails every read. + #[cfg(any(feature = "chargebee", feature = "paypal"))] + #[derive(Default)] + struct BillingSecrets { + map: StdMutex>, + fail: bool, + } + + #[cfg(any(feature = "chargebee", feature = "paypal"))] + #[async_trait] + impl SecretStore for BillingSecrets { + async fn get( + &self, + _c: &CompanyId, + key: &str, + ) -> crate::Result> { + if self.fail { + return Err(crate::error::OpenCompanyError::Store( + "the secret store is unreachable".into(), + )); + } + Ok(self + .map + .lock() + .unwrap() + .get(key) + .map(|v| crate::ports::types::SecretValue(v.clone()))) + } + async fn set( + &self, + _c: &CompanyId, + key: &str, + value: crate::ports::types::SecretValue, + ) -> crate::Result<()> { + self.map.lock().unwrap().insert(key.to_string(), value.0); + Ok(()) + } + } + + /// A company whose manifest allows exactly `grants`. + #[cfg(any(feature = "chargebee", feature = "paypal"))] + fn record_granting(grants: &[&str]) -> CompanyRecord { + let mut rec = record(); + rec.manifest.tools.allow = grants.iter().map(|g| g.to_string()).collect(); + rec + } + + /// The inert fixture deps, with a secret store and a "last known" connection. + #[cfg(any(feature = "chargebee", feature = "paypal"))] + fn billing_deps(dir: &std::path::Path, secrets: Arc) -> HarnessDeps { + let mut deps = deps_with_plan(dir, Arc::new(MockContext::default()), None, None); + deps.secrets = Some(secrets); + deps + } + + #[cfg(feature = "chargebee")] + #[tokio::test] + async fn chargebee_resolves_only_for_a_company_that_grants_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let secrets = Arc::new(BillingSecrets::default()); + secrets + .set( + &CompanyId::new("acme"), + crate::chargebee::types::SITE_SECRET, + crate::ports::types::SecretValue("acme-test".into()), + ) + .await + .expect("seed"); + secrets + .set( + &CompanyId::new("acme"), + crate::chargebee::types::API_KEY_SECRET, + crate::ports::types::SecretValue("cb_key".into()), + ) + .await + .expect("seed"); + let deps = billing_deps(dir.path(), secrets); + let pool = HarnessPool::new(); + + // Granted and configured: the credential resolves. + let granted = pool + .resolve_chargebee(&record_granting(&["chargebee"]), &deps) + .await + .expect("a granted, configured company resolves"); + assert_eq!(granted.site(), "acme-test"); + + // Same credentials, no grant. The store is untouched — the gate is the + // manifest, so a company that never opted in gets no tools however well + // configured the host happens to be. + assert!( + pool.resolve_chargebee(&record_granting(&[]), &deps) + .await + .is_none(), + "an ungranted company must resolve nothing" + ); + + // And a wildcard is not a grant: these tools send invoices to real + // people, so they are opted into by name rather than riding in on the + // `*` somebody set for file and shell tools. + assert!( + pool.resolve_chargebee(&record_granting(&["*"]), &deps) + .await + .is_none(), + "a catch-all grant must not confer chargebee" + ); + } + + #[cfg(feature = "chargebee")] + #[tokio::test] + async fn a_chargebee_store_hiccup_keeps_the_last_known_connection() { + // The distinction this pins: absence wires no tools, but a READ FAILURE + // keeps whatever was already resolved. Collapsing the two would drop a + // working company's billing tools mid-conversation on one bad read, and + // silently — the agent would simply stop being able to invoice. + let dir = tempfile::tempdir().expect("tempdir"); + let secrets = Arc::new(BillingSecrets { + fail: true, + ..Default::default() + }); + let mut deps = billing_deps(dir.path(), secrets); + let last_known = crate::harness::chargebee::TenantChargebee::resolve( + &(Arc::new(BillingSecrets { + map: StdMutex::new( + [ + ( + crate::chargebee::types::SITE_SECRET.to_string(), + "acme-test".to_string(), + ), + ( + crate::chargebee::types::API_KEY_SECRET.to_string(), + "cb_key".to_string(), + ), + ] + .into_iter() + .collect(), + ), + fail: false, + }) as Arc), + &CompanyId::new("acme"), + ) + .await + .expect("the seeded store reads") + .expect("both halves present"); + deps.chargebee = Some(last_known); + + let kept = pool_resolve_chargebee(&deps).await; + assert_eq!( + kept.map(|c| c.site().to_string()).as_deref(), + Some("acme-test"), + "a transient read failure must not disconnect a working integration" + ); + } + + #[cfg(feature = "chargebee")] + async fn pool_resolve_chargebee( + deps: &HarnessDeps, + ) -> Option { + HarnessPool::new() + .resolve_chargebee(&record_granting(&["chargebee"]), deps) + .await + } + + #[cfg(feature = "paypal")] + #[tokio::test] + async fn paypal_resolves_only_for_a_company_that_grants_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let secrets = Arc::new(BillingSecrets::default()); + for (key, value) in [ + (crate::company::paypal::CLIENT_ID_SECRET, "AY_id"), + (crate::company::paypal::CLIENT_SECRET_SECRET, "EL_secret"), + ] { + secrets + .set( + &CompanyId::new("acme"), + key, + crate::ports::types::SecretValue(value.into()), + ) + .await + .expect("seed"); + } + let deps = billing_deps(dir.path(), secrets); + let pool = HarnessPool::new(); + + assert!( + pool.resolve_paypal(&record_granting(&["paypal"]), &deps) + .await + .is_some(), + "a granted, configured company resolves" + ); + assert!( + pool.resolve_paypal(&record_granting(&[]), &deps) + .await + .is_none(), + "an ungranted company must resolve nothing" + ); + assert!( + pool.resolve_paypal(&record_granting(&["*"]), &deps) + .await + .is_none(), + "a catch-all grant must not confer paypal" + ); + } + + #[cfg(feature = "paypal")] + #[tokio::test] + async fn a_paypal_grant_with_no_credential_wires_nothing_rather_than_failing() { + // Fail closed: a manifest that grants `paypal` on a host where nobody + // has saved a credential must wire no tools, not tools that fail on + // first use — an agent that HAS a wallet tool tells the operator the + // balance is unavailable, rather than that it cannot read wallets. + let dir = tempfile::tempdir().expect("tempdir"); + let deps = billing_deps(dir.path(), Arc::new(BillingSecrets::default())); + assert!( + HarnessPool::new() + .resolve_paypal(&record_granting(&["paypal"]), &deps) + .await + .is_none() + ); + } +} diff --git a/src/harness/orchestrator.rs b/src/harness/built_in/orchestrator.rs similarity index 99% rename from src/harness/orchestrator.rs rename to src/harness/built_in/orchestrator.rs index 0a7d35405..4096ed334 100644 --- a/src/harness/orchestrator.rs +++ b/src/harness/built_in/orchestrator.rs @@ -4746,6 +4746,7 @@ mod tests { name: None, description: None, tier: tier.map(str::to_string), + harness: None, tools: Vec::new(), delegates_to: Vec::new(), context: None, diff --git a/src/harness/paypal.rs b/src/harness/built_in/paypal.rs similarity index 100% rename from src/harness/paypal.rs rename to src/harness/built_in/paypal.rs diff --git a/src/harness/planning.rs b/src/harness/built_in/planning.rs similarity index 100% rename from src/harness/planning.rs rename to src/harness/built_in/planning.rs diff --git a/src/harness/planning/test.rs b/src/harness/built_in/planning/test.rs similarity index 100% rename from src/harness/planning/test.rs rename to src/harness/built_in/planning/test.rs diff --git a/src/harness/policy.rs b/src/harness/built_in/policy.rs similarity index 100% rename from src/harness/policy.rs rename to src/harness/built_in/policy.rs diff --git a/src/harness/provider.rs b/src/harness/built_in/provider.rs similarity index 94% rename from src/harness/provider.rs rename to src/harness/built_in/provider.rs index 93c8da097..5142b8c1e 100644 --- a/src/harness/provider.rs +++ b/src/harness/built_in/provider.rs @@ -453,7 +453,7 @@ fn attach_tools( /// tool. Mirrors the shape openhuman's own `OpenHumanBackendModel` uses against /// the identical `/openai/v1` backend. static MANAGED_PROFILE: LazyLock = LazyLock::new(|| ModelProfile { - provider: Some("managed".to_string()), + provider: Some("openrouter".to_string()), modalities: Modalities { image_in: true, ..Modalities::default() @@ -835,7 +835,10 @@ impl ChatModel<()> for HostedProvider { impl HarnessModel for HostedProvider { fn telemetry_provider_id(&self) -> String { - "managed".to_string() + // `HostedProvider` is the platform-credentialed path by construction — + // it exists only where the env default supplied both endpoint and key — + // so its spend is the subscription's. + "subscription".to_string() } } @@ -880,36 +883,40 @@ pub async fn request_plan( tools: Vec, tool_choice: &ToolChoice, ) -> anyhow::Result { - let model = decl - .models - .get(abstract_model) - .cloned() - .unwrap_or_else(|| abstract_model.to_string()); + // Tier -> what this endpoint understands. The direct path talks to + // OpenRouter, which has never heard of `chat-v1`, so the tier is resolved + // here; the proxied path keeps the tier, which is what the platform's + // registry routes on. + let model = inference::model_for_tier(abstract_model, &decl.models, decl.is_proxied()); let url = format!("{}/chat/completions", decl.base_url.trim_end_matches('/')); let bearer = decl .bearer() .await .map_err(|e| anyhow::anyhow!("resolving the outbound inference credential: {e}"))?; - let headers = if decl.provider == "openrouter" { - vec![ - ("HTTP-Referer", OPENROUTER_REFERER.to_string()), - ("X-Title", OPENROUTER_TITLE.to_string()), - ] - } else if decl.provider == "managed" { - // Only `"managed"` is a TinyHumans-owned endpoint. The other three - // `INFERENCE_PROVIDERS` (`openrouter`, `openai_compatible`, `ollama` - // — see `company::types::INFERENCE_PROVIDERS`) are bring-your-own-key - // THIRD-PARTY endpoints (OpenAI, OpenRouter, DeepSeek, a - // self-hosted/local Ollama); sending them our `x-sdk-name` would leak - // which product a tenant is running to an operator who has no - // relationship with TinyHumans and gains nothing from knowing it. - // `openrouter` already gets its own attribution headers above — those - // are OpenRouter's own dashboard/rankings feature, unrelated to this. + let mut headers = Vec::new(); + + // OpenRouter's own attribution headers, on BOTH the proxied and the direct + // path: they identify the app in OpenRouter's dashboard and rankings, which + // is a feature we want either way and is unrelated to who is paying. + if inference::normalize_provider(&decl.provider) == "openrouter" { + headers.push(("HTTP-Referer", OPENROUTER_REFERER.to_string())); + headers.push(("X-Title", OPENROUTER_TITLE.to_string())); + } + + // The product-identity header goes ONLY to the platform's own endpoint — + // i.e. proxied OpenRouter. Every other resolution reaches a THIRD-PARTY + // endpoint (OpenRouter direct on the tenant's account, a self-hosted + // OpenAI-compatible server, a local Ollama); sending them our `x-sdk-name` + // would leak which product a tenant is running to an operator who has no + // relationship with TinyHumans and gains nothing from knowing it. + // + // Keyed on `is_proxied()` rather than the provider kind: after `managed`'s + // removal the kind no longer distinguishes our endpoint from OpenRouter's, + // and it is the endpoint — not the vocabulary — that this rule is about. + if decl.is_proxied() { let (name, value) = crate::product::product_identity_header(); - vec![(name, value.to_string())] - } else { - Vec::new() - }; + headers.push((name, value.to_string())); + } let mut body = serde_json::json!({ "model": model, "temperature": temperature, @@ -993,6 +1000,11 @@ pub struct TenantProvider { /// the config the last turn actually used (cost attribution follows the /// switch). slug: RwLock<&'static str>, + /// Which harness's config and credential slots this provider resolves + /// against. Two `built_in` harnesses on one company each get their own + /// provider, differing only in this — which is what lets one ride the + /// subscription while the other runs on a key of its own. + scope: inference::HarnessScope, } impl TenantProvider { @@ -1010,18 +1022,34 @@ impl TenantProvider { manifest, env_default, client: reqwest::Client::new(), - slug: RwLock::new("managed"), + // Replaced by the resolved slug on the first turn; until then the + // company is on the default it booted with. + slug: RwLock::new("subscription"), + scope: inference::HarnessScope::default(), } } + /// Points this provider at one named harness's own config and credential + /// slots. Without it, every provider reads the company's default harness. + pub fn with_scope(mut self, scope: inference::HarnessScope) -> Self { + self.scope = scope; + self + } + + /// The harness this provider resolves for. + pub fn harness_id(&self) -> &str { + &self.scope.id + } + /// Re-resolves the effective config from the secret store and updates the /// cached telemetry slug. Errors when no provider is configured at all. async fn resolve(&self) -> anyhow::Result { - let decl = inference::resolve_effective( + let decl = inference::resolve_effective_scoped( &self.company, &self.manifest, self.env_default.as_ref(), self.secrets.as_ref(), + &self.scope, ) .await .map_err(|e| anyhow::anyhow!("resolving inference config: {e}"))? @@ -1444,7 +1472,7 @@ mod tests { credential: Credential::None, extra_headers: Vec::new(), }); - assert_eq!(provider.telemetry_provider_id(), "managed"); + assert_eq!(provider.telemetry_provider_id(), "subscription"); } /// The exact `/openai/v1` staging response shape: reply text plus a standard @@ -1954,8 +1982,11 @@ mod tests { .contains(&("X-Title", OPENROUTER_TITLE.to_string())) ); - // An unmapped tier passes through unchanged. - let passthrough = request_plan( + // A tier the manifest does not map takes the shipped default rather than + // passing through as a bare tier name. It used to pass through, which + // worked only because the platform endpoint resolved tier names; this + // decl is DIRECT, and OpenRouter has never heard of `reasoning-v1`. + let defaulted = request_plan( &decl, "reasoning-v1", Vec::new(), @@ -1966,7 +1997,22 @@ mod tests { ) .await .expect("plan"); - assert_eq!(passthrough.model, "reasoning-v1"); + assert_eq!(defaulted.model, "deepseek/deepseek-v4-pro"); + + // A concrete slug is still forwarded untouched, so a caller can name any + // model in OpenRouter's catalog. + let explicit = request_plan( + &decl, + "anthropic/claude-sonnet-4.5", + Vec::new(), + 0.2, + None, + Vec::new(), + &ToolChoice::Auto, + ) + .await + .expect("plan"); + assert_eq!(explicit.model, "anthropic/claude-sonnet-4.5"); } #[tokio::test] @@ -1994,32 +2040,36 @@ mod tests { assert!(plan.headers.is_empty(), "no OpenRouter headers for Ollama"); } - /// The positive half of issue #376 (AC #1): `provider = "managed"` always - /// targets a TinyHumans-owned endpoint, so [`request_plan`] must attach - /// our `x-sdk-name: opencompany` product header alongside the tier's - /// other headers. + /// The positive half of issue #376 (AC #1): a **proxied** config targets a + /// TinyHumans-owned endpoint, so [`request_plan`] must attach our + /// `x-sdk-name: opencompany` product header alongside the tier's other + /// headers. + /// + /// After `managed`'s removal the provider *kind* no longer tells our + /// endpoint from OpenRouter's — the same `openrouter` kind reaches both. + /// `is_proxied()` is what distinguishes them, and this test and its negative + /// twin below pin both sides of that one bit. #[tokio::test] - async fn request_plan_attaches_the_product_header_for_managed() { + async fn request_plan_attaches_the_product_header_when_proxied() { let company = CompanyId::new("acme"); let secrets = MemSecrets::default(); let env = crate::company::inference::EnvDefault { base_url: "https://env.example/openai/v1".into(), credential: Credential::from_value("platform-key"), }; - // A hand-written `provider = "managed"` still resolves through the - // env default (mirrors `manifest_managed_inherits_env_credential` in - // `company::inference`'s own test suite) — this is the shape a real - // company manifest produces, not a synthetic decl. + // A keyless `openrouter` resolves through the env default — this is the + // shape a real company manifest produces, not a synthetic decl, and it + // is the config a company that has configured nothing runs on. let decl = inference::resolve_effective( &company, - &manifest_inference("managed"), + &manifest_inference("openrouter"), Some(&env), &secrets, ) .await .unwrap() - .expect("managed resolves via the env default"); - assert_eq!(decl.provider, "managed"); + .expect("keyless openrouter resolves via the env default"); + assert!(decl.is_proxied()); let plan = request_plan( &decl, @@ -2035,7 +2085,13 @@ mod tests { assert!( plan.headers .contains(&("x-sdk-name", "opencompany".to_string())), - "managed provider must carry the product header: {:?}", + "a proxied config must carry the product header: {:?}", + plan.headers + ); + assert!( + plan.headers + .contains(&("HTTP-Referer", OPENROUTER_REFERER.to_string())), + "and OpenRouter's own attribution rides the proxied path too: {:?}", plan.headers ); } @@ -2046,14 +2102,17 @@ mod tests { /// host an operator points at — OpenAI, DeepSeek, a self-hosted proxy, /// …). Sending them our product identity would tell a company we have no /// relationship with which product a tenant is running, for no benefit to - /// anyone. Only `"managed"` (see the test above) may ever carry the - /// header. + /// anyone. Only a **proxied** config (see the test above) may ever carry + /// the header — and note the first case here is the *same provider kind* as + /// that test, differing only in holding a tenant key. That is precisely the + /// distinction this rule now turns on. #[tokio::test] async fn request_plan_never_attaches_the_product_header_for_third_party_providers() { let company = CompanyId::new("acme"); let secrets = MemSecrets::default(); - // openrouter: gets ITS OWN attribution headers, never ours. + // openrouter DIRECT (the tenant's own key): gets ITS OWN attribution + // headers, never ours. let mut or_manifest = manifest_inference("openrouter"); or_manifest.models = BTreeMap::from([("chat-v1".to_string(), "deepseek/deepseek-chat".to_string())]); diff --git a/src/harness/publish.rs b/src/harness/built_in/publish.rs similarity index 98% rename from src/harness/publish.rs rename to src/harness/built_in/publish.rs index 818de10c4..e72abcd98 100644 --- a/src/harness/publish.rs +++ b/src/harness/built_in/publish.rs @@ -684,11 +684,21 @@ pub fn capture_body( source: &str, _inferred: ArtifactKind, ) -> std::io::Result { + // Route by size before reading, so the read is bounded by the prose cap: a + // file already past it goes straight to bytes, and a file within the cap is + // read whole and probed in place. + let over_cap = file + .metadata() + .map(|meta| meta.len() > MAX_ARTIFACT_BODY_BYTES as u64) + .unwrap_or(false); let bytes = std::fs::read(file)?; - if bytes.len() <= MAX_ARTIFACT_BODY_BYTES - && let Ok(text) = String::from_utf8(bytes.clone()) - { - return Ok(PublishPayload::Text(text)); + if !over_cap && std::str::from_utf8(&bytes).is_ok() { + // The borrowed probe validates in place; the move below reuses the same + // buffer, so a file within the cap is never copied. The probe's `Err` + // arm is unreachable here, hence the `expect` on a just-validated vec. + return Ok(PublishPayload::Text( + String::from_utf8(bytes).expect("probed utf-8 in place"), + )); } Ok(PublishPayload::Bytes { mime: mime_guess::from_path(source) diff --git a/src/harness/publish/test.rs b/src/harness/built_in/publish/test.rs similarity index 100% rename from src/harness/publish/test.rs rename to src/harness/built_in/publish/test.rs diff --git a/src/harness/publish_turn_test.rs b/src/harness/built_in/publish_turn_test.rs similarity index 99% rename from src/harness/publish_turn_test.rs rename to src/harness/built_in/publish_turn_test.rs index 0bb0601db..2c8a34b4b 100644 --- a/src/harness/publish_turn_test.rs +++ b/src/harness/built_in/publish_turn_test.rs @@ -309,6 +309,7 @@ fn brain_with( extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(ops.clone()), diff --git a/src/harness/repo.rs b/src/harness/built_in/repo.rs similarity index 100% rename from src/harness/repo.rs rename to src/harness/built_in/repo.rs diff --git a/src/harness/repo/test.rs b/src/harness/built_in/repo/test.rs similarity index 100% rename from src/harness/repo/test.rs rename to src/harness/built_in/repo/test.rs diff --git a/src/harness/run_trace.rs b/src/harness/built_in/run_trace.rs similarity index 100% rename from src/harness/run_trace.rs rename to src/harness/built_in/run_trace.rs diff --git a/src/harness/run_turn.rs b/src/harness/built_in/run_turn.rs similarity index 57% rename from src/harness/run_turn.rs rename to src/harness/built_in/run_turn.rs index e89e9ab14..897c37ac0 100644 --- a/src/harness/run_turn.rs +++ b/src/harness/built_in/run_turn.rs @@ -17,24 +17,31 @@ use crate::Result; use crate::company::steer::SteerControl; use crate::harness::run_trace::RunTraceSink; use crate::harness::{HarnessDeps, HarnessPool, TurnOutcome}; -use crate::ports::types::CompanyId; +use crate::ports::types::{CompanyId, CompanyRecord}; use crate::runtime::delegation::RunTurn; -/// The harness's [`RunTurn`]: re-attaches [`HarnessDeps`] onto each pool turn. -pub struct HarnessRunTurn<'a> { - pool: &'a HarnessPool, - deps: &'a HarnessDeps, +/// The built-in harness's [`RunTurn`]: re-attaches [`HarnessDeps`] onto each +/// pool turn. +/// +/// Holds its pool and deps by `Arc` rather than by reference so it can live in +/// a [`HarnessRouter`](crate::harness::router::HarnessRouter) alongside the +/// other lanes. A company running two `built_in` harnesses has one of these per +/// harness, each over its own pool and its own provider — which is the whole +/// point of naming more than one. +pub struct HarnessRunTurn { + pool: Arc, + deps: Arc, } -impl<'a> HarnessRunTurn<'a> { - /// Wraps a pool + deps for one cycle's worth of turns. - pub fn new(pool: &'a HarnessPool, deps: &'a HarnessDeps) -> Self { +impl HarnessRunTurn { + /// Wraps a pool + deps as one harness lane. + pub fn new(pool: Arc, deps: Arc) -> Self { Self { pool, deps } } } #[async_trait] -impl RunTurn for HarnessRunTurn<'_> { +impl RunTurn for HarnessRunTurn { async fn run( &self, company: &CompanyId, @@ -43,7 +50,7 @@ impl RunTurn for HarnessRunTurn<'_> { chat_id: Option<&str>, ) -> Result { self.pool - .run(company, agent_id, message, self.deps, chat_id) + .run(company, agent_id, message, &self.deps, chat_id) .await } @@ -58,7 +65,7 @@ impl RunTurn for HarnessRunTurn<'_> { ) -> Result { self.pool .run_steered( - company, agent_id, message, self.deps, control, chat_id, run_sink, + company, agent_id, message, &self.deps, control, chat_id, run_sink, ) .await } @@ -72,7 +79,22 @@ impl RunTurn for HarnessRunTurn<'_> { run_sink: Option>, ) -> Result { self.pool - .run_steered_background(company, agent_id, message, self.deps, control, run_sink) + .run_steered_background(company, agent_id, message, &self.deps, control, run_sink) .await } + + async fn run_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + ) -> Result { + self.pool + .run_background(company, agent_id, message, &self.deps) + .await + } + + async fn ensure(&self, company: &CompanyRecord) -> Result<()> { + self.pool.ensure(company, &self.deps).await + } } diff --git a/src/harness/search.rs b/src/harness/built_in/search.rs similarity index 98% rename from src/harness/search.rs rename to src/harness/built_in/search.rs index 2da47ab03..083b52239 100644 --- a/src/harness/search.rs +++ b/src/harness/built_in/search.rs @@ -585,7 +585,14 @@ fn source_domain(url: &str) -> Option { /// call, which is what stops a poisoned snippet forging the closing marker and /// speaking as the harness. Same construction as the workspace tools' fence. fn fence_nonce() -> String { - crate::ports::generate_id() + // Same construction as workspace_tools' fence: drawn from the OS CSPRNG, + // not [`crate::ports::generate_id`], because the fence's sole property is + // unforgeability. A predictable nonce lets a search result that happens + // to cite a prior fence token forge the closing marker and speak as the + // harness. + let mut bytes = [0u8; 16]; + getrandom::fill(&mut bytes).expect("the OS CSPRNG is unavailable; cannot mint a content fence"); + bytes.iter().map(|byte| format!("{byte:02x}")).collect() } /// Render the citation block the agent sees. diff --git a/src/harness/search_turn_test.rs b/src/harness/built_in/search_turn_test.rs similarity index 99% rename from src/harness/search_turn_test.rs rename to src/harness/built_in/search_turn_test.rs index f5342ae51..a4898ebfe 100644 --- a/src/harness/search_turn_test.rs +++ b/src/harness/built_in/search_turn_test.rs @@ -267,6 +267,7 @@ async fn harness( extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(meter.clone()), diff --git a/src/harness/skills.rs b/src/harness/built_in/skills.rs similarity index 93% rename from src/harness/skills.rs rename to src/harness/built_in/skills.rs index 798432a97..dbb9f1038 100644 --- a/src/harness/skills.rs +++ b/src/harness/built_in/skills.rs @@ -53,6 +53,19 @@ mod naming; pub use naming::{DESCRIBE_SKILL_TOOL, LIST_SKILLS_TOOL, READ_SKILL_RESOURCE_TOOL}; +/// Whether `slug` is a safe directory name for `skills//`: the same +/// `^[a-z0-9][a-z0-9-]*$` shape the page tools enforce. Anything else — a +/// traversal (`..`), a path separator, a dotfile — would escape the scratch +/// tree via [`Path::join`], so it is refused wherever a slug enters. +fn valid_slug(slug: &str) -> bool { + let mut chars = slug.chars(); + match chars.next() { + Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {} + _ => return false, + } + chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') +} + /// One agent's effective, enabled skill set, materialized on disk so OpenHuman's /// skill read tools can scan it. pub struct EffectiveSkills { @@ -125,8 +138,20 @@ impl EffectiveSkills { } // 2. Apply operator deltas: disables drop, enabled custom docs supersede. + // A delta whose slug is not a safe directory name is skipped — a + // traversal slug must never reach the `skills_out.join(slug)` write + // below (the console validates at write time; this is the belt for a + // row that predates that check or lands through a non-console path). let mut disabled: HashSet = HashSet::new(); for delta in deltas { + if !valid_slug(&delta.slug) { + log::warn!( + "[harness][skills] skipping a skill delta whose slug is not a safe \ + directory name: {:?}", + delta.slug + ); + continue; + } if !delta.enabled { disabled.insert(delta.slug.clone()); continue; @@ -670,6 +695,40 @@ mod tests { assert_eq!(written, body); } + /// A delta whose slug is not a safe directory name must never reach the + /// `skills_out.join(slug)` write: `..` would escape the scratch tree, and + /// the console validates slugs at write time, so such a row is either a + /// pre-check row or a non-console write — skip it either way. + #[test] + fn a_traversal_slug_delta_is_skipped_not_written_outside() { + let ws = tempfile::tempdir().unwrap(); + let body = "---\nname: Escape\ndescription: Should never land\n---\n\n# Escape\n"; + + let eff = EffectiveSkills::materialize( + ws.path().to_path_buf(), + None, + &[], + &[delta("..", true, Some(body))], + ) + .unwrap(); + + // The bogus delta contributed nothing to the effective set… + assert_eq!(eff.docs.len(), with_baseline(&[])); + assert!(eff.docs.iter().all(|doc| doc.slug != "..")); + // …and its write never escaped the scratch tree: `skills/..` resolves + // to the workspace root, which is where the escaped SKILL.md would have + // landed. + assert!( + !ws.path().join("SKILL.md").exists(), + "the traversal delta wrote nothing outside the skills tree" + ); + // Only the baseline dirs materialize — no `..` directory inside either. + assert_eq!( + std::fs::read_dir(ws.path().join("skills")).unwrap().count(), + eff.docs.len() + ); + } + #[test] fn custom_doc_supersedes_company_body() { let src = tempfile::tempdir().unwrap(); diff --git a/src/harness/skills/naming.rs b/src/harness/built_in/skills/naming.rs similarity index 100% rename from src/harness/skills/naming.rs rename to src/harness/built_in/skills/naming.rs diff --git a/src/harness/skills/naming/test.rs b/src/harness/built_in/skills/naming/test.rs similarity index 100% rename from src/harness/skills/naming/test.rs rename to src/harness/built_in/skills/naming/test.rs diff --git a/src/harness/steer.rs b/src/harness/built_in/steer.rs similarity index 100% rename from src/harness/steer.rs rename to src/harness/built_in/steer.rs diff --git a/src/harness/steps.rs b/src/harness/built_in/steps.rs similarity index 100% rename from src/harness/steps.rs rename to src/harness/built_in/steps.rs diff --git a/src/harness/tool_dispatcher.rs b/src/harness/built_in/tool_dispatcher.rs similarity index 100% rename from src/harness/tool_dispatcher.rs rename to src/harness/built_in/tool_dispatcher.rs diff --git a/src/harness/toolbelt.rs b/src/harness/built_in/toolbelt.rs similarity index 97% rename from src/harness/toolbelt.rs rename to src/harness/built_in/toolbelt.rs index 42ae3ec29..0f0a8c03c 100644 --- a/src/harness/toolbelt.rs +++ b/src/harness/built_in/toolbelt.rs @@ -525,6 +525,22 @@ pub fn media_tools(backend: &MediaBackend, workspace: &Path) -> Vec cursor = parent_id, + None => break, + } + } + } + // Where this lands, so the "already taken" check below asks about the // path that will actually exist rather than about one of its halves. // The node resolved strictly inside the home, so it always has a parent diff --git a/src/harness/workspace_tools/lifecycle/test.rs b/src/harness/built_in/workspace_tools/lifecycle/test.rs similarity index 95% rename from src/harness/workspace_tools/lifecycle/test.rs rename to src/harness/built_in/workspace_tools/lifecycle/test.rs index d7f68ed94..a7e0362b6 100644 --- a/src/harness/workspace_tools/lifecycle/test.rs +++ b/src/harness/built_in/workspace_tools/lifecycle/test.rs @@ -615,6 +615,53 @@ async fn a_name_and_a_parent_can_change_in_one_call() { assert_eq!(node.parent_id.as_deref(), Some("f-archive")); } +/// A folder must never land inside its own subtree: `archive` → `archive/deep` +/// would make the tree unreadable for every agent from then on. Refused before +/// the store is asked to create the cycle. +#[tokio::test] +async fn a_folder_cannot_be_moved_into_its_own_subfolder() { + let home = own_home("acme").await; + let mine = home.home_id().await; + home.store + .create( + &home.company, + &folder("f-deep", "deep", Some("f-archive")), + None, + ) + .await + .unwrap(); + + // By path, into the descendant. + let out = home + .renamer() + .execute(json!({ + "path": "Agents/ceo/archive", + "new_parent": "Agents/ceo/archive/deep", + })) + .await + .unwrap(); + assert!(out.is_error, "{}", text(&out)); + assert!( + text(&out).contains("unreadable"), + "the refusal says the tree would be unreadable: {}", + text(&out) + ); + + // By id, into itself — the same guard at the end of the ancestry walk. + let out = home + .renamer() + .execute(json!({ "id": "f-archive", "new_parent": "Agents/ceo/archive" })) + .await + .unwrap(); + assert!(out.is_error, "{}", text(&out)); + + // The tree is untouched. + assert_eq!( + home.node("f-archive").await.parent_id.as_deref(), + Some(mine.as_str()) + ); +} + /// A binary node renames like any other — the port moves the payload with it. #[tokio::test] async fn a_binary_node_can_be_renamed_and_keeps_its_payload() { diff --git a/src/harness/workspace_turn_test.rs b/src/harness/built_in/workspace_turn_test.rs similarity index 99% rename from src/harness/workspace_turn_test.rs rename to src/harness/built_in/workspace_turn_test.rs index 968d74c22..82389bb4e 100644 --- a/src/harness/workspace_turn_test.rs +++ b/src/harness/built_in/workspace_turn_test.rs @@ -273,6 +273,7 @@ async fn harness( extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, diff --git a/src/harness/cap_publish_test.rs b/src/harness/cap_publish_test.rs index 6d9d1bd20..5f7eef407 100644 --- a/src/harness/cap_publish_test.rs +++ b/src/harness/cap_publish_test.rs @@ -299,6 +299,7 @@ fn deps_for(base_url: String, dir: &std::path::Path) -> (HarnessDeps, Arc extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(ops.clone()), diff --git a/src/harness/cap_turn_test.rs b/src/harness/cap_turn_test.rs index 269935e81..020217da4 100644 --- a/src/harness/cap_turn_test.rs +++ b/src/harness/cap_turn_test.rs @@ -301,6 +301,7 @@ fn deps_for(base_url: String, dir: &std::path::Path) -> (HarnessDeps, Arc extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(ops.clone()), diff --git a/src/harness/lanes.rs b/src/harness/lanes.rs new file mode 100644 index 000000000..7b7508206 --- /dev/null +++ b/src/harness/lanes.rs @@ -0,0 +1,257 @@ +//! Turning a company's declared `[[harness]]` set into the engines that serve +//! it. +//! +//! One place decides, for every declared harness, whether this host can run it +//! and what runs it — so the runtime builder does not grow a second opinion +//! about which agent lands where. +//! +//! ## One pool per `built_in` harness +//! +//! Each `built_in` harness gets its own [`HarnessPool`] and its own +//! [`HarnessDeps`], differing in exactly two fields: the provider (scoped to +//! that harness's config and credential slots) and +//! [`serves`](HarnessDeps::serves), which narrows the pool to the agents bound +//! to it. +//! +//! The narrowing is what makes one-pool-per-harness affordable. Without it every +//! pool would build every agent, so a ten-agent roster across three harnesses +//! would stand up thirty live agents — each holding a model client — to use ten. +//! +//! ## What is declared but not runnable +//! +//! An `acp` harness has no engine here yet: its transports live in the desktop +//! shell and the runner lane, and neither is wired into the server build. Rather +//! than silently routing those agents somewhere else, the harness is recorded as +//! unavailable with the reason, and a turn bound to it fails saying so. Falling +//! back would be the worst outcome available — the turn would succeed, on a +//! model and a credential nobody chose. + +use std::collections::HashSet; +use std::sync::Arc; + +use crate::company::Harness; +use crate::company::inference::{EnvDefault, HarnessScope}; +use crate::harness::built_in::provider::TenantProvider; +use crate::harness::built_in::run_turn::HarnessRunTurn; +use crate::harness::built_in::{HarnessDeps, HarnessPool}; +use crate::ports::SecretStore; +use crate::ports::types::{CompanyId, CompanyRecord}; +use crate::runtime::delegation::RunTurn; + +/// The engines a company's declared harnesses resolve to on this host. +pub struct Lanes { + /// Agents the **default** harness serves, when the company declares more + /// than one. `None` means the whole roster — the single-harness case. + pub default_serves: Option>, + /// Every lane beyond the default: its harness id and the engine serving it. + pub lanes: Vec<(String, Arc)>, + /// Declared harnesses this host cannot run, and why. + pub unavailable: Vec<(String, String)>, +} + +/// Which agents are bound to `harness_id`, given the company's default. +fn agents_on(record: &CompanyRecord, harness_id: &str, default_harness: &str) -> HashSet { + let mut ids: HashSet = record + .manifest + .agents + .iter() + .filter(|a| a.harness.as_deref().unwrap_or(default_harness) == harness_id) + .map(|a| a.id.clone()) + .collect(); + // A console-created (overlay) teammate has no manifest row and therefore no + // harness binding — `overlay_agent_to_manifest` hardcodes `harness: None` — + // so every overlay runs on the default harness. Fold them into the default + // lane's serve set, or a multi-harness company would build them on no pool + // at all: the default pool's `serves` would exclude them, no other lane + // claims them, and the roster would silently drop a teammate the console + // is still showing. + if harness_id == default_harness { + ids.extend(record.overlay_agents.iter().map(|a| a.id.clone())); + } + ids +} + +/// Builds the lanes for `record`, given the deps the **default** harness runs +/// on. +/// +/// Returns no lanes at all for a company that declares no `[[harness]]` (or +/// declares exactly one): there is nothing to route, and the caller keeps its +/// single pool untouched. That is the path every existing company takes. +pub fn build( + record: &CompanyRecord, + base: &HarnessDeps, + secrets: Arc, + env_default: Option, +) -> Lanes { + let declared = record.manifest.effective_harnesses(); + let default_harness = record.manifest.default_harness_id(); + + if declared.len() <= 1 { + return Lanes { + default_serves: None, + lanes: Vec::new(), + unavailable: Vec::new(), + }; + } + + let mut lanes = Vec::new(); + let mut unavailable = Vec::new(); + + for harness in declared.iter().filter(|h| h.id != default_harness) { + match harness.kind.as_str() { + "built_in" => lanes.push(( + harness.id.clone(), + built_in_lane( + record, + base, + &secrets, + env_default.clone(), + harness, + &default_harness, + ), + )), + // The ACP transports are supplied by the desktop shell (a stdio + // subprocess) and the runner lane (a socket); a server build has + // neither, so there is nothing to hand a turn to. + "acp" => unavailable.push(( + harness.id.clone(), + "it is an ACP harness and this build has no ACP transport wired — \ + run it from the desktop app, or bind these agents to a `built_in` harness" + .to_string(), + )), + other => unavailable.push(( + harness.id.clone(), + format!("`{other}` is not a harness kind this build knows how to run"), + )), + } + } + + Lanes { + default_serves: Some(agents_on(record, &default_harness, &default_harness)), + lanes, + unavailable, + } +} + +/// One `built_in` lane: its own pool, over deps carrying its own provider and +/// narrowed to the agents bound to it. +fn built_in_lane( + record: &CompanyRecord, + base: &HarnessDeps, + secrets: &Arc, + env_default: Option, + harness: &Harness, + default_harness: &str, +) -> Arc { + // Its own `[harness.inference]`, else the company-level `[inference]` — the + // caller cannot pick, because only the harness knows whether it declared + // one. + let manifest_inference = harness + .inference + .clone() + .unwrap_or_else(|| record.manifest.inference.clone()); + + let provider = Arc::new( + TenantProvider::new( + record.id.clone(), + secrets.clone(), + manifest_inference, + env_default, + ) + .with_scope(HarnessScope::named(&harness.id)), + ); + + let mut deps = base.clone(); + deps.provider = provider; + deps.serves = Some(agents_on(record, &harness.id, default_harness)); + + Arc::new(HarnessRunTurn::new( + Arc::new(HarnessPool::new()), + Arc::new(deps), + )) +} + +/// The company id a lane set was built for. Exposed so a caller can assert it +/// wired the lanes it thinks it did. +pub fn company_of(record: &CompanyRecord) -> &CompanyId { + &record.id +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ports::types::OverlayAgent; + + /// A two-harness company with a console-created overlay teammate. + fn record() -> CompanyRecord { + let manifest: crate::company::CompanyManifest = toml::from_str( + r#" +[company] +name = "Acme" + +[[agent]] +id = "ceo" +role = "Chief Executive" + +[[agent]] +id = "researcher" +role = "Researcher" +harness = "deep" + +[[harness]] +id = "embedded" +kind = "built_in" +default = true + +[[harness]] +id = "deep" +kind = "built_in" +"#, + ) + .expect("valid manifest"); + CompanyRecord { + id: CompanyId::new("acme"), + manifest, + ledger: Vec::new(), + lifecycle: "running".to_string(), + overlay_agents: vec![OverlayAgent { + id: "writer".into(), + name: "Writer".into(), + role: "Content Writer".into(), + description: None, + tools: Vec::new(), + }], + overlay_desk_members: Vec::new(), + overlay_desk_order: Vec::new(), + overlay_desks: Vec::new(), + overlay_workflows: Vec::new(), + overlay_budgets: Vec::new(), + overlay_policy: None, + overlay_desk_tools: Default::default(), + disabled_workflows: Vec::new(), + template_provenance: None, + setup: None, + } + } + + /// The default lane serves the whole default-bound roster **including** + /// every overlay teammate, whose only harness is the default. + #[test] + fn the_default_lane_serves_every_overlay_agent() { + let rec = record(); + let default = agents_on(&rec, "embedded", "embedded"); + assert!(default.contains("ceo")); + assert!(!default.contains("researcher"), "bound to the deep lane"); + assert!( + default.contains("writer"), + "a console-created teammate runs on the default harness" + ); + + // And the named lane must not claim it — the overlay is nobody's but + // the default's. + let deep = agents_on(&rec, "deep", "embedded"); + assert!(deep.contains("researcher")); + assert!(!deep.contains("writer")); + assert!(!deep.contains("ceo")); + } +} diff --git a/src/harness/mod.rs b/src/harness/mod.rs index ff20ec829..311b4b607 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -1,55 +1,38 @@ -//! WS4 — openhuman embedded as a library (the harness). +//! Harnesses: the execution engines a company's agents run their turns on. //! -//! This module supersedes the out-of-process OpenHuman seam -//! (`src/openhuman/{launcher,rpc,tools,channel}.rs`, JSON-RPC behind -//! `openhuman-rpc`) with **direct library embedding** of `vendor/openhuman` -//! (`openhuman_core`): one openhuman [`Agent`](oh::agent::Agent) per manifest -//! `[[agent]]`, wired with memory, an inference provider, an approval policy, -//! and a workspace through [`AgentBuilder`](oh::agent::AgentBuilder). +//! A **harness** is one answer to "what actually runs this agent's turn". A +//! company declares a named set of them in `company.toml` and binds each agent +//! to one, so a single company can put its researcher on a deep reasoning model, +//! its bulk workers on a cheap one, and its coding agent on the operator's own +//! Claude Code — the last of which needs no credential from us at all. //! -//! Compiled only under `feature = "openhuman"`. The default build links none of -//! it and keeps its offline, echo-brained behaviour. +//! Two kinds ship: //! -//! ## Layout +//! * [`built_in`] — the embedded OpenHuman/tinyagents loop, in this process, +//! against an inference provider the harness itself declares. Everything +//! under `built_in/` is *one harness implementation*, not "the harness". +//! * [`acp`] — an external agent driven over the Agent Client Protocol, either +//! a subprocess on the operator's machine or a runner that dialed in. //! -//! * [`build`] — manifest `[[agent]]` → `AgentBuilder`. -//! * [`provider`] — hosted Medulla [`Provider`] + a `MockProvider` for tests. -//! * [`memory`] — [`OcMemory`](memory::OcMemory): openhuman `Memory` over the -//! opencompany [`ContextStore`](crate::ports::ContextStore). -//! * [`policy`] — [`ApprovalPolicy`](policy::ApprovalPolicy): `[policy]` → -//! openhuman `ToolPolicy`. -//! * [`cost`] — [`TurnCost`](oh::agent::cost::TurnCost) → ledger + usage meter. +//! Both are [`RunTurn`](crate::runtime::delegation::RunTurn) implementations, +//! which is the whole point: the company cycle asks for a turn and does not +//! learn which engine served it. //! -//! ## Flagged seams +//! ## Transitional re-exports //! -//! * **Group-chat / desk routing** is opencompany's job (openhuman is -//! single-agent). v1 is single-responder; the full ops `chat` handler that -//! resolves a desk's members and journals the `AgentReply` is WS3. -//! -//! Live turn cost is **wired**: [`CompanyAgent::run`] reads the completed turn's -//! token/cost totals from openhuman's public -//! [`Agent::last_turn_usage`](oh::agent::Agent::last_turn_usage) accessor and -//! [`HarnessPool::run`] records them through [`cost::record_turn_cost`]. Usage -//! only reaches the ledger/meter when the provider reports it — the -//! [`HostedProvider`](provider::HostedProvider) parses it off the wire; the -//! offline [`MockProvider`](provider::MockProvider) does not, so test turns stay -//! inert. +//! `built_in`'s contents were previously declared directly here, so the glob +//! below keeps every `crate::harness::X` path resolving while callers migrate to +//! `crate::harness::built_in::X`. It is deliberately a re-export rather than a +//! rename in place: the move commit that created this file changed no content, +//! and the paths are updated separately. + +pub mod acp; +pub mod built_in; +pub mod lanes; +pub mod router; + +pub use built_in::*; -/// A `RunTurn` over an ACP agent (the `acp` feature). -/// -/// Gated because nothing in a default build can reach it: the endpoint that -/// would drive it lives behind the same feature, and `/acp` is a reserved -/// prefix that 404s without it. Compiling it unconditionally meant a surface -/// that no lane ran and no route served — see issue #475. -#[cfg(feature = "acp")] -pub mod acp_run_turn; -/// Issue #775: the fail-closed shell audit wrapper — one intent line appended -/// (and fsynced) *before* a command runs, refusing the command outright when -/// that append fails. Pairs with the host-owned, per-agent sink -/// [`toolbelt::shell_audit`] resolves. See [`audit`]. -pub mod audit; -pub mod brain; -pub mod build; /// Issue #989 (Part 2a of #926): end-to-end proof that a **chat** turn which /// pauses at its tool-iteration cap runs the same #244 unpublished-work scan /// and nudge the task-dispatch path (`run_task`) already gets — and that a @@ -61,6930 +44,30 @@ mod cap_publish_test; /// bubble saying so, and the notice never reaches memory. Test-only. #[cfg(test)] mod cap_turn_test; -pub mod capability_budget; -#[cfg(feature = "chargebee")] -pub mod chargebee; -mod checkpoint; -pub mod composio; -/// Issue #410: how a Composio action catalogue is narrowed and rendered for an -/// agent, and why every cut it makes describes itself. Pure and un-gated (the -/// live tools are behind `composio`, which CI never *runs*) — see -/// [`composio_catalog`]. -pub mod composio_catalog; -/// End-to-end proof that #410's narrowable, self-describing Composio listing is -/// reachable from a real turn on two large toolkits — the harness, the grant -/// gate, the approval policy and the Composio client are all real; only the -/// model's choices and the Composio backend are scripted. Test-only. -#[cfg(all(test, feature = "composio"))] -mod composio_turn_test; -/// Issue #416: the confined turn — an ephemeral agent with no tools, no company -/// memory and no delegation, for a question that is about one object rather than -/// about the company. See [`confine`]. -pub mod confine; -pub mod cost; -/// Hosted embeddings compute for the in-pod memory engine's meaning tier (188c2). -/// Needs the `tinycortex` crate's `EmbeddingBackend` trait, so it links only when -/// both the harness (`openhuman`) and the memory engine (`tinycortex`) are built. -#[cfg(feature = "tinycortex")] -pub mod embeddings; -/// Hosting (TinyHosts): the per-company connection and the agent tools over it. -/// The keys it reads live in `company::hosting`, which is compiled in every -/// build — the console's Hosting settings write them whether or not this -/// harness exists to use them. -pub mod hosting; -/// End-to-end proof of issue #988: a turn really does get -/// [`MAX_TOOL_ITERATIONS`](build::MAX_TOOL_ITERATIONS) tool rounds instead of the -/// vendored ten, and a budget-armed turn's in-turn -/// [`BudgetStopHook`](oh::agent::stop_hooks::BudgetStopHook) halts it when it -/// outruns its money — distinguishably from an iteration-cap pause. Test-only. -#[cfg(test)] -mod iteration_cap_turn_test; -pub mod ledger_tools; -pub mod lifecycle; -pub mod mcp; -pub mod mcp_probe; -pub mod memory; -pub mod memory_loop; -pub mod orchestrator; /// Agent-authored internal dashboard pages: `pages_list` / `pages_read` / /// `pages_write` / `pages_delete` over `Pages//` in the same /// [`crate::ports::workspace::WorkspaceStore`], with `pages_write` compiling /// `Page.tsx` to `Page.compiled.mjs` via `swc_core`. See /// `docs/spec/runtime/pages.md`. pub mod pages_tools; -/// Chargebee billing tools (issue #788), wired per company from its own -/// SecretStore. Always compiled so the credential resolution and the fail-closed -/// decision are testable at default features; only the tools are gated. -/// PayPal wallet + transaction tools (issue #789), wired per company from its -/// own SecretStore. Always compiled so credential resolution and the -/// fail-closed decision are testable at default features. -#[cfg(feature = "paypal")] -pub mod paypal; -/// Issue #337: the planning station — one tool-less model call per card entering -/// `planning`, with the host gathering the evidence and verifying every -/// prerequisite the model claims. See [`planning`]. -pub mod planning; -pub mod policy; -pub mod provider; -/// Issue #244: `publish_artifact` — the only way a workspace file becomes a -/// deliverable — plus the staging queue the brain drains, the bounded workspace -/// scan that detects unpublished work, and the follow-up nudge's prompt. See -/// [`publish`]. -pub mod publish; -/// End-to-end proof that #244's `publish_artifact` is reachable from a real -/// dispatch, that a re-run extends by identity, and — the part nothing shorter -/// than a real turn loop can show — that the follow-up nudge fires **once**, -/// records a decline, and can never fail the run it follows. Test-only. -#[cfg(test)] -mod publish_turn_test; -/// Issue #245, agent half: `repo_checkout` / `repo_pr` behind an explicit -/// `repo` grant — a **confined** working tree cloned out of the host's mirror -/// (a full object copy, then every reference back to the mirror severed), plus -/// the per-turn ledger that deletes it again. See [`repo`]. -pub mod repo; /// First-run company setup's pass: one tool-less model call that designs a /// company's starting team from three answers. See [`roster_build`]. pub mod roster_build; -pub mod run_trace; -pub mod run_turn; -pub mod search; -/// End-to-end proof that the #238 `web_search` tool is reachable from a real -/// turn — the harness, the grant gates, the approval policy, the cap and the -/// meter are all real; only the model's choices and the search backend's -/// responses are scripted. Test-only. -#[cfg(test)] -mod search_turn_test; -pub mod skills; +/// Issue #1032: the in-turn spend brake — the +/// [`StopHook`](oh::agent::stop_hooks::StopHook) wrapper that makes a budget +/// halt observable to this crate, and the [`SpendHalt`] record the +/// operator-facing notice is composed from. Read by +/// [`TurnOutcome::halted_for_spend`](built_in::TurnOutcome::halted_for_spend). pub mod spend; +/// Issue #1032: end-to-end proof that a turn stopped by its in-turn spend +/// brake **says so** — and says something different from a turn that paused at +/// its step cap. Test-only. #[cfg(test)] mod spend_halt_turn_test; -pub mod steer; -pub mod steps; -pub mod tool_dispatcher; -pub mod toolbelt; -pub mod triage; -/// Issue #661 (M7): `read_workflow` / `update_workflow` / `delete_workflow` — -/// the agent's way to fix or retire a workflow instead of only ever creating -/// another one beside it. Kept out of `orchestrator.rs` (already the largest -/// file in `src/harness/`) because the three share a handle, a guard and a set -/// of refusals with each other rather than with anything there. See -/// [`workflow_admin`]. -pub mod workflow_admin; -/// Issue #339: the staging queue the orchestrator's `run_workflow` / -/// `create_workflow` tools push a workflow reference onto and the -/// [`HarnessBrain`] drains at the end of a dispatch, so a card that built or -/// Issue #580: the workflow builder pass — turns a `workflow`-deliverable card's -/// plan into a proposed graph that lands In Review for approval. Modeled on the -/// planning station (one card, one tool-less model call, one settled outcome), -/// but it mints an attempt row because building the workflow is the card's work. -/// See [`workflow_build`]. -pub mod workflow_build; -/// ran a workflow can link to it. See [`workflow_refs`]. -pub mod workflow_refs; -/// End-to-end proof that an agent granted `files` and **not** `shell` can write -/// a relative path on a company that has never run — the #409 provisioning gap, -/// which only exists before anything has created the agent's workspace. Covers -/// a manifest teammate and a runtime overlay teammate, and pins that a traversal -/// out of a provisioned sandbox is still refused. Test-only. -#[cfg(test)] -mod workspace_provision_turn_test; -pub mod workspace_tools; -/// End-to-end proof that the #237 workspace tools are reachable from a real -/// turn, with only the model's choices stubbed. Test-only. -#[cfg(test)] -mod workspace_turn_test; - -pub use brain::HarnessBrain; - -use std::collections::{HashMap, HashSet}; -use std::path::PathBuf; -use std::sync::Arc; - -use openhuman_core::openhuman as oh; -use tokio::sync::{Mutex, RwLock}; - -use oh::agent::Agent; - -use crate::harness::provider::HarnessModel; - -use crate::company::Agent as ManifestAgent; -use crate::company::Policy; -use crate::company::mcp::McpServerDecl; -use crate::company::steer::{SteerAction, SteerControl}; -use crate::error::OpenCompanyError; -use crate::harness::cost::{TurnUsage, record_turn_cost}; -use crate::harness::mcp_probe::McpFailureQueue; -use crate::harness::orchestrator::DelegationQueue; -use crate::harness::policy::{ApprovalPolicy, ApprovalRequestQueue}; -use crate::ports::skills_state::{SkillState, SkillStateStore}; -use crate::ports::types::{ - BudgetOverride, CompanyId, CompanyRecord, OverlayAgent, OverlayDesk, OverlayDeskMember, - PolicyOverride, TurnStep, -}; -use crate::ports::{ - ArtifactStore, CompanyStore, ContextStore, EventLog, FactStore, SecretStore, TaskStore, - UsageMeter, -}; -use crate::runtime::builder::agent_scoped_grants; - -/// Shared dependencies every harness-built agent draws on. -#[derive(Clone)] -pub struct HarnessDeps { - /// The inference model shared across a company's agents. A [`HarnessModel`] - /// is a tinyagents [`ChatModel<()>`](tinyagents::harness::model::ChatModel) - /// plus the telemetry slug the cost hook reads live per turn; it upcasts to - /// `Arc>` at the openhuman `AgentBuilder::chat_model` seam. - pub provider: Arc, - /// Stable provider slug attributed to usage samples (e.g. `managed`). - pub provider_slug: String, - /// Context store backing every agent's [`OcMemory`](memory::OcMemory). - pub context: Arc, - /// Company store the cost hook appends ledger entries to. - pub store: Arc, - /// Optional usage meter (WS5 seam); `None` skips usage sampling. - pub meter: Option>, - /// Root under which per-agent workspace directories are created - /// (`{root}/{company}/{agent}/workspace`). - pub workspace_root: PathBuf, - /// Whether each private agent workspace is initialized as a Git repository - /// and checkpointed after tool calls. Host-level `[workspace]` config owns - /// this switch; false preserves the pre-checkpoint behavior exactly. - pub workspace_git_enabled: bool, - /// The **instance data root** the shell audit sink hangs off, resolved - /// through [`DataLayout::agent_audit_dir`](crate::store::DataLayout::agent_audit_dir) - /// to `companies//audit//` (issue #775). - /// - /// Carried as its own field rather than derived from - /// [`workspace_root`](Self::workspace_root)`.parent()` on purpose. The two - /// are siblings under one data root today, and an implicit - /// `workspace_root/..` would make a security boundary depend on a directory - /// relationship nobody declared — the ambient-context coupling this codebase - /// keeps getting bitten by. The audit sink is where it is because a caller - /// said so. - /// - /// It must never be inside `workspace_root`: the agent workspace is also the - /// `workspace_only` `SecurityPolicy` root the file tools enforce, so a sink - /// under it is a policy-*permitted* write target for the very agent it - /// records. - pub audit_root: PathBuf, - /// Optional model/tier applied to every agent, overriding the per-agent - /// `tier` → model mapping. Set from the resolved hosted-inference model so - /// the whole roster addresses the configured workload (e.g. `chat-v1`). - /// `None` keeps each agent's tier-derived default. - pub model_override: Option, - /// The company's task board, so a [`TaskDispatched`] cycle can load the - /// dispatched card and write its result back. `None` off the task path (the - /// chat brain leaves the board untouched). - /// - /// [`TaskDispatched`]: crate::ports::types::CompanyEvent::TaskDispatched - pub tasks: Option>, - /// The company's artifact store, so a dispatched card's output is recorded - /// as a versioned artifact (#187) instead of only as note text. `None` - /// leaves the board's behaviour exactly as before — the note is still - /// written either way, so an unwired artifact store loses nothing that - /// existed previously. - pub artifacts: Option>, - /// The company's ledgers, so an agent can read what has already been - /// decided, goaled or ruled out, record what it decides, and declare an axis - /// nobody anticipated. `None` builds no ledger tools at all — which is right - /// for a path with no company behind it, and is what every construction site - /// that predates them does. - pub ledgers: Option>, - /// The company's ledgers as they stood when the agent was built, for the - /// prompt catalogue. - /// - /// Resolved to **data** before deps construction because `build_agent` is - /// synchronous, the same shape the MCP servers already take. A ledger - /// declared mid-run is therefore reachable by every tool immediately (the - /// `ledger` argument is checked against the live registry at call time) and - /// appears in the *prompt* only from the next build — which is the honest - /// limit: system prompts are assembled once, and nothing can retroactively - /// edit one already in flight. - pub ledger_registry: crate::ledger::Registry, - /// The company's skill-delta store, so a built agent can see its effective - /// skill set (company-dir skills ∪ operator deltas ∪ custom docs) as read - /// tools + a prompt catalogue. `None` leaves the agent skill-less (the chat - /// path off the skills seam builds no skill surface). - /// - /// See [`skills`](crate::harness::skills) — this is the read-only catalogue - /// slice; skill *execution* is deferred. - pub skills: Option>, - /// The company's source directory (`companies/`), whose `skills/` - /// subtree supplies the committed skill bundles unioned into the effective - /// set. `None` surfaces only the operator deltas. - pub skills_source_dir: Option, - /// The repo-level shared skill library (`skills/*/SKILL.md`), the same set - /// the console's registry tab browses. Used only to heal pre-fix registry - /// installs, whose stored snapshot is a one-line stub — see - /// [`EffectiveSkills::materialize`](crate::harness::skills::EffectiveSkills::materialize). - /// Empty only when the host serves no shared skill library, where a stub - /// simply stays as it is; a platform-provisioned runtime otherwise receives - /// the library the application state loaded, same as the serve path. - pub skills_registry: Arc<[crate::company::SkillDoc]>, - /// The company's effective MCP servers (issue #50), resolved to **data** - /// (manifest `[[mcp_server]]` ∪ the runtime index, with each server's - /// outbound credential materialized to - /// [`AuthMaterial`](crate::company::mcp::AuthMaterial)) before deps - /// construction. `build_agent` is synchronous but the - /// [`SecretStore`](crate::ports::SecretStore) is async, so the runtime - /// builder resolves these ahead of time; each agent then filters the set by - /// its `mcp:*` tool grants. Empty leaves the agent with no MCP bridge tools. - pub mcp_servers: Vec, - /// Install-wide default MCP servers (issue #527), carried so the live - /// re-resolution in [`Harness::resolve_effective_mcp`] merges the same three - /// layers the boot-time resolution did. Without it a console edit would - /// re-resolve to manifest ∪ runtime and silently drop every default. - pub default_mcp_servers: Vec, - /// The company's durable [`FactStore`], surfaced to the orchestrator agent - /// through the `query_company` read tool (issue #53). `None` leaves the - /// orchestrator without the facts half of its insight surface (the chat path - /// off the orchestrator seam wires nothing). - pub facts: Option>, - /// The company's [`EventLog`], surfaced to the orchestrator agent through - /// the `query_company` read tool for recent-activity context (issue #53). - /// `None` leaves the orchestrator without the recent-events half. - pub events: Option>, - /// The shared delegation queue the orchestrator's `spawn_task` / - /// `delegate_to_desk` tools push onto and the [`HarnessBrain`] drains after - /// an orchestrator turn (issue #53). A [`DelegationQueue`] is a cheap shared - /// handle; cloning `HarnessDeps` shares one queue between the tools built - /// into the agent and the brain that drains it. Default is an empty queue. - pub delegations: DelegationQueue, - /// The shared handle to the company's [`WorkflowRunner`](crate::ports::WorkflowRunner), - /// so the orchestrator's `run_workflow` tool can reach the runner that is - /// itself built *from* these deps (issue #67). The runtime builder threads an - /// empty handle here, builds the [`HarnessWorkflowRunner`](crate::workflows::HarnessWorkflowRunner) - /// from a deps clone, then fills the shared cell — so the orchestrator agent - /// (built later from a clone of these deps) reaches it at turn time. The cell - /// holds a [`Weak`](std::sync::Weak), so deps↔runner is not a strong cycle. - /// Default (and any build with no runner) leaves it empty and the tool - /// reports workflow execution is not wired. - pub workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle, - /// The shared MCP failure queue the `OcMcpCallTool` decorator pushes onto and - /// the [`HarnessBrain`] drains after a turn (the error-hardening cell). Same - /// cheap-shared-handle pattern as [`Self::delegations`]; every string it - /// carries is scrubbed at the source. Default is an empty queue. - pub mcp_failures: McpFailureQueue, - /// The shared publish queue the `publish_artifact` tool stages onto and the - /// [`HarnessBrain`] drains at the end of a dispatch (issue #244). Same - /// cheap-shared-handle pattern as [`Self::mcp_failures`], and for the same - /// structural reason: tools are built **once per agent** while the card - /// varies **per dispatch**, so a tool cannot hold a task id or a store and - /// has to hand its work to something that does. - /// - /// Default is an empty queue, which simply means nothing is ever published - /// — every path degrades to "this task produced no artifact", which is a - /// legitimate outcome rather than a failure. - pub pending_publishes: crate::harness::publish::PendingPublishQueue, - /// The shared queue the orchestrator's `run_workflow` / `create_workflow` - /// tools stage a workflow reference onto and the [`HarnessBrain`] drains at - /// the end of a dispatch (issue #339) — the workflow half of a card's - /// output link. - /// - /// Same cheap-shared-handle pattern as [`Self::pending_publishes`], and for - /// the same structural reason: the tools are built **once per agent** while - /// the card varies **per dispatch**, so a tool cannot hold a task id and has - /// to hand its work to something that does. - /// - /// Default is an empty queue, which simply means no card ever links to a - /// workflow — the stamp falls back to the attempt's trace, which is a - /// complete answer rather than a missing one. - pub workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue, - /// The bounded, in-process cache the orchestrator's `run_workflow` tool fills - /// with each successful run's node output and the `read_run_output` companion - /// reads back (issue #418) — so a preview the run summary clipped is - /// reachable within the same turn. - /// - /// Same cheap-shared-handle pattern as [`Self::workflow_refs`]: the run tool - /// that stores and the read tool that serves are built in one `build_agent` - /// pass off the same deps clone, so they share one cache. Default is an empty - /// cache; nothing durable rides on it (the console run drawer is the durable - /// record), so a fresh process simply starts with nothing to read back. - pub run_outputs: crate::harness::orchestrator::RunOutputCache, - /// The DURABLE, console-facing per-node run output store (issue #596) — - /// distinct from [`Self::run_outputs`] above, which is the in-process, - /// evictable agent cache. The workflow runner persists each settled run's - /// bounded node output here so a *past* run reopened from History shows what - /// every node produced. `None` (the default build, and every unwired test) - /// degrades the persist to a no-op, exactly like [`Self::events`]. - pub run_output_store: Option>, - /// Issue #274's per-workflow snapshot ring, so the orchestrator's - /// `update_workflow` / `delete_workflow` tools (issue #661, M7) write - /// through the same undo-and-cascade path the console's `PUT`/`DELETE` - /// routes do — an agent edit is recoverable on exactly the terms an - /// operator's is. - /// - /// `None` (the default build, and every unwired test) makes those two tools - /// refuse rather than degrade, unlike [`Self::events`]. The asymmetry is the - /// point: a missing journal loses an audit line, while a missing revision - /// store loses the only copy of the graph being overwritten. - pub workflow_revisions: Option>, - /// The shared approval-request queue every agent's [`ApprovalPolicy`] pushes - /// a `RequireApproval` decision onto and the [`HarnessBrain`] drains after a - /// turn, parking each request through - /// [`CycleHost::park_effect`](crate::ports::brain::CycleHost::park_effect) - /// so it reaches the operator's Approvals page (issue #172). Same - /// cheap-shared-handle pattern as [`Self::delegations`]; the default is an - /// empty queue, which simply means nothing is ever parked. - pub approval_requests: ApprovalRequestQueue, - /// The company's [`SecretStore`], so [`HarnessPool::ensure`] can **re-resolve** - /// the effective MCP server set on each call and rebuild the roster when a - /// console add/remove/enable-toggle changes it — the MCP-freshness fix (a - /// runtime-added server reaches the agent on its next turn, no restart). - /// `None` (default/tests) keeps the boot-resolved [`Self::mcp_servers`] - /// static, exactly as before. - pub secrets: Option>, - /// The per-company SSRF allowlist for the `web` toolbelt (Cell A), from the - /// manifest `[tools].web_allowed_domains`. Empty (the default) is *open - /// mode* — all public hosts allowed — while OpenHuman's upstream `url_guard` - /// still rejects private/loopback/link-local/metadata IPs regardless. A - /// non-empty list is strict (only those hosts + subdomains); `"*"` is an - /// explicit allow-all-public wildcard. Threaded verbatim into - /// [`toolbelt::web_tools`](crate::harness::toolbelt::web_tools). - pub web_allowed_domains: Vec, - /// The capability-tier filter applied to each agent's assembled tool vector - /// (Cell A seam). [`AllowAll`](crate::harness::toolbelt::CapabilityFilter::AllowAll) - /// (the default) is identity. When [`Self::plan`] is set, - /// [`HarnessPool::ensure`] overwrites this per turn with the tenant's - /// resolved filter; when the plan is `None` this stays the no-plan - /// fallback/test override. - pub capabilities: toolbelt::CapabilityFilter, - /// The company's source directory (`companies/`), from which a - /// workflow's `sub_workflow` nodes resolve a child by `workflow_id` - /// (`workflows/.toml`). Distinct from - /// [`Self::skills_source_dir`](Self::skills_source_dir) so the two seams stay - /// independent even though both currently derive from the same `seed_dir`. - /// `None` (default/tests, and platform-provisioned tenants with nothing on - /// disk) keeps the loud `UnwiredResolver`, so a reached `sub_workflow` node - /// fails clearly instead of resolving nothing. - pub workflow_source_dir: Option, - /// The tenant's capability tier plan (issue #108). `None` (the default) - /// leaves gating **off** — byte-identical to Cell A, [`Self::capabilities`] - /// is used verbatim. When set, [`HarnessPool::ensure`] resolves a per-tenant, - /// per-period, fail-closed [`CapabilityFilter`](toolbelt::CapabilityFilter) - /// from the [`UsageMeter`] before each turn and installs it on the roster it - /// builds. Resolved from the manifest `[plan]` section by the runtime builder. - pub plan: Option, - /// The MANAGED media-generation backend (issue #109). `None` (the default at - /// every construction site) fails closed — no image/video tools are wired. - /// Only the production runtime builder sets it, from - /// [`media_backend_from_env`](crate::harness::provider::media_backend_from_env) - /// (env-only — never a tenant secret). When `Some` **and** a company - /// explicitly grants `media`, [`build::build_agent`] wires the - /// [`toolbelt::media_tools`]; a grant with no credential wires nothing and - /// warns. - pub media: Option, - /// The per-tenant Composio configuration (issue #110). `None` (the default - /// at every construction site) fails closed — no Composio tools are wired. - /// [`HarnessPool::ensure`] re-resolves it each turn (folded into the roster - /// fingerprint) so a console token set/rotate takes effect next turn with no - /// restart. Only wired when a company **explicitly** grants `composio` **and** - /// a credential can be obtained: the company's own token under - /// [`composio::TOKEN_KEY`](crate::harness::composio::TOKEN_KEY) if it has one, - /// else this instance's platform identity. With neither, no tools are wired — - /// never a borrowed identity. - pub composio: Option, - - /// The per-company Chargebee connection (issue #788). `None` (the default at - /// every construction site) fails closed — no billing tools are wired. - /// Resolved from that company's own secret store, never from the - /// environment: two companies on one host bill two different sites. - /// `HarnessPool::ensure` re-resolves it each turn, so a key set or rotated in - /// the console takes effect next turn with no restart. - #[cfg(feature = "chargebee")] - pub chargebee: Option, - - /// The per-company PayPal connection (issue #789). `None` fails closed — - /// no wallet tools are wired. Resolved from that company's own secret store - /// and re-resolved each turn, like `chargebee`. - #[cfg(feature = "paypal")] - pub paypal: Option, - - /// The per-company hosting connection. `None` (the default at every - /// construction site) fails closed — no hosting tools are wired. Resolved - /// from that company's own secret store and re-resolved each turn, like - /// `chargebee`: two companies on one host deploy to two different hosting - /// accounts, and a deployment publishes files to the internet under the - /// account's own name. - pub hosting: Option, - /// The MANAGED web-search backend (issue #238). `None` (the default at every - /// construction site but the production runtime builder) **fails closed** — - /// no `web_search` tool is wired and agents behave exactly as before. - /// - /// Set by the runtime builder from - /// [`search_backend_from_env`](crate::harness::provider::search_backend_from_env) - /// (env-only — never a tenant secret) with the company's - /// `[tools].search_daily_calls` cap applied. When `Some` **and** a company - /// **explicitly** grants `search` (never via `*`), [`build::build_agent`] - /// wires [`search::search_tools`]; a grant with no credential wires nothing - /// and warns, media's shape exactly. - /// - /// The handle carries the company's shared daily-call ledger, so cloning - /// these deps across a roster gives every agent of the company one budget - /// rather than one each. - pub search: Option, - /// Issue #111 — the shared registry of in-flight, steerable runs. The - /// [`HarnessBrain`] registers a dispatched task / desk delegation here before - /// running it (and installs the steer stop-hook over the slot's control), so - /// an operator can pause / cancel / redirect it mid-flight. The **same** - /// handle is threaded onto the [`CompanyRuntime`](crate::company::runtime::CompanyRuntime) - /// so the operator steer routes reach it. A cheap shared handle (like - /// [`delegations`](Self::delegations)); the default is an empty registry, - /// which simply lists nothing and rejects every steer as `not in flight`. - pub steer: crate::company::steer::InflightRegistry, - /// Issue #383 — the shared set of cancellable workflow runs. The - /// orchestrator's `run_workflow` tool mints its run context through this, so - /// an agent-initiated run appears in the same map the console's cancel route - /// reads and is stoppable like any other. The runtime builder threads in the - /// same handle it puts on the [`CompanyRuntime`](crate::company::CompanyRuntime); - /// the default is a private map nothing else can see, which simply means the - /// tool's runs are not cancellable. - pub run_supervisor: crate::runtime::RunSupervisor, - /// Issue #170 — the ports an `output` node's `destination` needs to route a - /// finished workflow's report to a person or a channel (mail handle, inbox, - /// user directory, wired channels), bundled so this struct grows one field - /// rather than four. - /// - /// Read post-engine by - /// [`deliver_outputs`](crate::workflows::delivery::deliver_outputs) — never - /// by the engine, which knows nothing about destinations. `None` (the - /// default at every construction site but the production runtime builder) - /// **fails closed and loud**: nothing is sent and the run result carries a - /// `failed` row saying delivery is not wired, so an authored destination can - /// never quietly do nothing. - pub delivery: Option, - /// Issue #237 — the company's shared workspace note tree, so agents can - /// read (and, under an explicit `workspace` grant, revise) the operator's - /// standards and playbooks instead of guessing at them. - /// - /// The same [`WorkspaceStore`](crate::ports::WorkspaceStore) handle the - /// console's REST/GraphQL surface writes through, so an operator edit is - /// visible to the next agent turn with no rebuild — the tools hold no - /// snapshot and hit the store per call. `None` (the default at every - /// construction site but the production runtime builder) **fails closed**: - /// no workspace tools are wired and agents behave exactly as before. - pub workspace: Option>, - /// Issue #245, agent half — the company's [`RepoManager`], so an agent that - /// explicitly grants `repo` can check a bound repository out and read a - /// pull request. `None` (the default at every construction site but the - /// production runtime builder) **fails closed**: no repository tools are - /// wired and agents behave exactly as before. - /// - /// [`RepoManager`]: crate::runtime::RepoManager - pub repos: Option>, - /// The company's bound repositories, resolved to **data** before deps - /// construction — the `mcp_servers` doctrine, and for the same reason: - /// [`build::build_agent`] is synchronous while reading the binding index is - /// async, and the tool descriptions name what is bound so a model does not - /// have to guess. Empty means nothing is bound, which is also what makes a - /// `repo` grant with no bindings wire nothing and warn. - pub repo_bindings: Vec, - /// The shared per-turn ledger of checkouts and diff spills, so the - /// [`CheckoutJanitor`](brain::CheckoutJanitor) claimed at each entry point - /// can delete them however the turn ends. - /// - /// Same cheap-shared-handle pattern as [`Self::pending_publishes`], and for - /// the same structural reason: the tools are built **once per agent** while - /// the deletion boundary is **per turn**. Default is an empty ledger, which - /// simply means nothing is ever recorded for deletion — the boot sweep is - /// the backstop. - pub checkouts: repo::CheckoutLedger, -} - -/// A minimal [`HarnessDeps`] for tests that only care about **workflow-tool -/// wiring**: which namespaces a `tool_call` can reach, and why the others cannot. -/// -/// Only the inputs [`workflow_tool_wiring`](crate::workflows::caps) actually -/// reads are parameters — the meter and plan (which resolve the capability -/// filter per company and spend) and the static filter itself. `search` is -/// pinned to `None`, because a deployment with no managed search backend is the -/// shape issue #874 is about. Everything else is the cheapest inert default, so -/// a test asserting on wiring does not have to name thirty fields that cannot -/// affect the answer. -/// -/// Shared rather than copied: the same fixture backs the runtime-level wiring -/// tests and the `tool-slugs` route test, so both ask about one deployment shape. -#[cfg(test)] -pub(crate) fn workflow_wiring_deps( - runtime: &crate::CompanyRuntime, - meter: Option>, - capabilities: toolbelt::CapabilityFilter, - plan: Option, -) -> HarnessDeps { - HarnessDeps { - ledgers: None, - ledger_registry: Default::default(), - provider: Arc::new(provider::MockProvider::default()), - provider_slug: "mock".to_string(), - context: runtime.context.clone(), - store: runtime.store.clone(), - meter, - workspace_root: std::env::temp_dir(), - workspace_git_enabled: false, - audit_root: std::env::temp_dir(), - model_override: None, - tasks: None, - artifacts: None, - skills: None, - skills_source_dir: None, - skills_registry: Arc::from([]), - mcp_servers: Vec::new(), - default_mcp_servers: Vec::new(), - facts: None, - events: None, - delegations: orchestrator::DelegationQueue::default(), - workflow_runner: orchestrator::WorkflowRunnerHandle::default(), - mcp_failures: mcp_probe::McpFailureQueue::default(), - pending_publishes: publish::PendingPublishQueue::default(), - workflow_refs: workflow_refs::WorkflowRefQueue::default(), - run_outputs: orchestrator::RunOutputCache::default(), - run_output_store: None, - workflow_revisions: None, - approval_requests: policy::ApprovalRequestQueue::default(), - secrets: None, - web_allowed_domains: Vec::new(), - capabilities, - workflow_source_dir: None, - plan, - media: None, - composio: None, - #[cfg(feature = "chargebee")] - chargebee: None, - #[cfg(feature = "paypal")] - paypal: None, - hosting: None, - // The staging shape in issue #874: `searchCredentialConfigured: false`. - search: None, - steer: crate::company::steer::InflightRegistry::default(), - run_supervisor: crate::runtime::RunSupervisor::default(), - delivery: None, - workspace: None, - repos: None, - repo_bindings: Vec::new(), - checkouts: repo::CheckoutLedger::default(), - } -} - -/// One live openhuman agent, keyed by its manifest id. -pub struct CompanyAgent { - /// The manifest agent id. - pub agent_id: String, - /// The manifest agent's human-readable role. - pub role: String, - /// This teammate's manifest `budget_usd_daily` cap, carried onto the roster - /// so the dispatch gate in [`HarnessPool::run_inner`] can read it without - /// re-loading the manifest per turn (issue #304). - /// - /// `None` for an uncapped teammate — and for every overlay teammate, which - /// carries no per-agent cap in v1. - pub budget_usd_daily: Option, - /// The embedded openhuman session. A [`Mutex`] because a `turn` takes - /// `&mut self` and one agent must serialise its own turns. - agent: Mutex, -} - -/// The graceful reply returned when a turn yields the transient empty-response -/// class twice — so chat never shows a bare "Couldn't send" for a model hiccup. -const GRACEFUL_EMPTY_REPLY: &str = "Sorry — I hit a temporary model hiccup and couldn't produce a reply. Please resend your message."; - -/// The operator-facing notice returned when the plan-level total token ceiling -/// (issue #188) is reached — a hard dispatch refusal, so no model call is made. -/// Surfaced as the turn's reply on every dispatch path (operator chat, task, -/// steered/background), since they all funnel through -/// [`HarnessPool::run_inner`](HarnessPool::run_inner). -const TOTAL_BUDGET_EXHAUSTED_NOTICE: &str = - "Token budget for this period is exhausted — dispatch paused until the period resets."; - -/// The operator-facing notice returned when one teammate has spent its manifest -/// `budget_usd_daily` (issue #304) — a hard dispatch refusal for that teammate -/// only, made before any model call. -/// -/// Deliberately a *visible refusal* rather than a silent no-op, and deliberately -/// per-teammate: the rest of the company keeps running, and the operator is told -/// which desk stopped, what its cap is, and when it comes back. There is no -/// per-call unit to park at turn level — an inference turn is not a tool call — -/// so a notice is the honest answer, mirroring -/// [`TOTAL_BUDGET_EXHAUSTED_NOTICE`]. -fn agent_budget_exhausted_notice(agent_id: &str, cap_usd: f64) -> String { - format!( - "{agent_id} has reached its daily spend cap of ${cap_usd:.2} — dispatch to this teammate \ - is paused until the cap resets at 00:00 UTC. Other teammates are unaffected." - ) -} - -/// The classification of a single `agent.turn` attempt, for the retry wrapper. -enum AttemptOutcome { - /// A non-empty reply. - Reply(String), - /// The transient empty-response class (an empty/blank reply, or the model's - /// "empty response" error) — retryable. - Empty, - /// A hard error (budget/auth/build/etc.) — propagated loudly, never swallowed. - Hard(OpenCompanyError), -} - -/// The result of a completed turn: the reply text plus the scrubbed -/// [`TurnStep`] timeline folded from the turn's progress stream. -/// -/// The steps are per-bubble: the operator bubble carries the orchestrator's -/// steps, a delegated desk bubble carries that desk lead's steps. They ride the -/// wire on [`OutboundMessage::steps`](crate::ports::types::OutboundMessage) and -/// are **never** written to memory ([`HarnessPool::run`] persists -/// `outcome.reply` only). -#[derive(Debug, Clone)] -pub struct TurnOutcome { - /// The agent's reply text. - pub reply: String, - /// The scrubbed, folded processing steps (empty for a memory-served or - /// tool-less turn — the zero-steps tell). - pub steps: Vec, - /// Whether this turn **paused at its tool-iteration cap** rather than - /// finishing what it set out to do (issue #926). - /// - /// A capped turn is not an error and never has been: openhuman stops the - /// tool loop, makes one extra tools-disabled call asking the model for a - /// resumable "Done so far / Next steps" checkpoint, and returns that as an - /// ordinary `Ok(reply)`. So the reply reads like a finished answer, and - /// nothing in the text, the steps or the error channel distinguishes "I - /// answered you" from "I ran out of steps mid-task" — which is exactly what - /// the operator could not tell. - /// - /// Read from openhuman's public - /// [`Agent::last_turn_hit_cap`](oh::agent::Agent::last_turn_hit_cap) while - /// the agent lock is still held, the same under-lock idiom - /// [`read_turn_usage`] uses. `false` on every path that returns an outcome - /// **without** running a model turn (the two pre-turn budget refusals, the - /// ACP fold) — a refusal is not a pause, and labelling one as a cap hit - /// would tell the operator to reply "continue" to a turn that never ran. - pub hit_iteration_cap: bool, - /// The in-turn **spend halt**, when one stopped this turn (issue #1032). - /// - /// `Some` exactly when the teammate declared a `budget_usd_daily`, the - /// [`SpendStopHook`](crate::harness::spend::SpendStopHook) armed for it - /// fired, and the turn therefore stopped short of the answer it was working - /// towards. `None` on every other path, including every turn by a teammate - /// who declared no budget — no hook is installed for them, so there is - /// nothing that could have halted them. - /// - /// A **separate** field from [`hit_iteration_cap`](Self::hit_iteration_cap) - /// rather than another reading of it, because the two are different - /// outcomes needing different operator actions: a step pause is resumable - /// with "continue", a spend halt means the work costs more than its budget - /// allows and asking again just spends more. #988 pinned that they are - /// distinguishable — a budget halt reads `last_turn_hit_cap() == false`, - /// because the run paused *below* `max_tool_iterations` — which is why this - /// could not be folded into the existing flag. - /// - /// Carries the figures rather than a bare `bool` so the notice can say what - /// was spent against which cap, and names the teammate so a chain of turns - /// cannot report a number the operator has no way to attribute. - pub halted_for_spend: Option, -} - -/// What one in-turn spend halt cost, and whose cap it was measured against -/// (issue #1032). -/// -/// The figures are the ones this crate already owns: the cap is -/// [`CompanyAgent::turn_spend_cap_usd`], and the spend is the sum of the -/// [`TurnUsage::cost_usd`](crate::harness::cost::TurnUsage::cost_usd) totals the -/// turn already reports. Deliberately **not** parsed out of the vendored hook's -/// `reason` string, which is a developer-facing trace line whose shape is -/// upstream's to change. -/// -/// `agent` is carried because one operator bubble can cover a responder turn, a -/// desk turn and a relay turn, each with its own cap. The iteration-cap notice -/// declines to name a number for exactly that reason; naming the teammate is -/// what makes a number attributable, and is why this one can be quoted where -/// that one could not. -#[derive(Debug, Clone, PartialEq)] -pub struct SpendHalt { - /// The teammate whose cap was reached. - pub agent: String, - /// What that teammate's turn had spent when the brake fired, in USD. - /// - /// Can exceed [`cap_usd`](Self::cap_usd): the brake fires *between* tool - /// iterations, so the call that crossed the line has already been paid for. - pub spent_usd: f64, - /// The cap it was measured against, in USD — the teammate's declared - /// `budget_usd_daily`. - pub cap_usd: f64, -} - -impl CompanyAgent { - /// Runs one turn against this agent, returning its reply text and the - /// per-attempt token/cost totals. - /// - /// **Empty-response hardening (the error-hardening cell)**: the hosted brain - /// occasionally returns a transient empty completion, which openhuman - /// surfaces as an error. Rather than letting the operator see a bare - /// "Couldn't send", this wrapper retries **once**; if the second attempt is - /// still empty it returns a graceful, scrubbed message instead of an `Err`. - /// **Non-transient** errors (budget, auth, build) still propagate loudly — no - /// blanket swallow. Every attempt's usage is returned so the cost hook meters - /// what the model actually consumed (a burnt empty attempt still costs - /// tokens). - /// - /// The usage is read from each just-completed turn via openhuman's public - /// [`Agent::last_turn_usage`](oh::agent::Agent::last_turn_usage) accessor - /// while the agent lock is still held. An offline provider that reports no - /// usage yields a zero [`TurnUsage`], which the cost hook treats as inert. - /// - /// **Activity-trace**: this is the one site holding `&mut Agent`, so it is - /// where the turn's [`AgentProgress`](oh::agent::progress::AgentProgress) - /// stream is captured. A per-turn `mpsc` channel is attached via - /// [`Agent::set_on_progress`](oh::agent::Agent::set_on_progress); an - /// always-draining collector task buffers every event so the turn loop never - /// blocks on a full channel; and after the turn (both attempts share the one - /// channel) the sink is detached, the collector joined, and the events folded - /// into the scrubbed [`TurnOutcome::steps`] by - /// [`steps::fold_steps`](crate::harness::steps::fold_steps). The sink is - /// per-turn *local* — deliberately not a [`HarnessDeps`] field — so parallel - /// turns never collide. - pub async fn run(&self, message: &str) -> crate::Result<(TurnOutcome, Vec)> { - self.run_with_steer(message, None, None, None).await - } - - /// Runs one turn with an optional operator **steer** control installed - /// (issue #111). - /// - /// When `steer` is `Some`, a [`SteerStopHook`](crate::harness::steer::SteerStopHook) - /// over the shared control is installed around the turn via - /// [`with_stop_hooks`](oh::agent::stop_hooks::with_stop_hooks). OpenHuman - /// fires stop hooks **between** tool-loop iterations (never mid-tool-call), - /// so an operator pause / cancel / redirect halts the turn gracefully at the - /// next iteration boundary. The control is `Box::pin`ned at the task-local - /// scope boundary to avoid the nested-scope stack-overflow trap. - /// - /// When a steer is pending after the first attempt yields the transient - /// empty-response class, the one-shot retry is **skipped** — a cancel (or - /// pause) issued before any text is produced must not silently restart the - /// work. With no steer this is byte-identical to the pre-#111 `run`. - /// - /// When `run_sink` is `Some`, the same collector also writes each step - /// through to the [`RunStore`](crate::ports::RunStore) as it arrives, so a - /// dispatched card's trace is durable *during* the run rather than only - /// after it (issue #242). The await lives in the collector task, never in - /// the model loop, so a slow store slows only trace persistence. `None` - /// (chat turns, workflow nodes, every test) is byte-identical to the prior - /// buffer-only behaviour. - pub async fn run_with_steer( - &self, - message: &str, - steer: Option<&SteerControl>, - stream: Option, - run_sink: Option>, - ) -> crate::Result<(TurnOutcome, Vec)> { - // Per-turn progress sink + an always-draining collector, so a burst of - // events never blocks the turn loop on a full channel. - // - // When `stream` is `Some`, the collector *tees* each event live onto the - // transient [`turn_stream`](crate::turn_stream) bus as it arrives — - // mirroring OpenHuman's `spawn_progress_bridge` — so the console renders - // the tool timeline while the turn is still running. The same events are - // still buffered and folded into the durable `TurnStep`s below, so the - // live view and the final reply timeline are byte-identical. With `None` - // (background turns, non-`openhuman` build) this is exactly the prior - // buffer-only behaviour. - let (tx, mut rx) = tokio::sync::mpsc::channel::(1024); - let collector = tokio::spawn(async move { - let mut events = Vec::new(); - let mut seq: u64 = 0; - // Mirrors `fold_steps`' thinking-run coalescing so the live timeline - // emits the same "Thinking" rows the final folded one does. - let mut thinking_open = false; - while let Some(event) = rx.recv().await { - if let Some(ctx) = &stream - && let Some(frame) = steps::stream_event_from(&event, seq, &mut thinking_open) - { - crate::turn_stream::publish( - &ctx.company, - frame - .with_agent(ctx.agent_id.clone()) - .with_chat(ctx.chat_id.clone()), - ); - seq += 1; - } - // Durable half (#242): persist the step before moving on, so a - // process killed mid-run keeps every step written so far. - if let Some(sink) = &run_sink { - sink.record(&event).await; - } - events.push(event); - } - events - }); - - let mut agent = self.agent.lock().await; - agent.set_on_progress(Some(tx)); - - // Two hooks, both fired by openhuman between tool-loop iterations: - // - // * the **steer** hook, only when an operator control is provided (#111); - // * the **budget** hook, only when this teammate declares a - // `budget_usd_daily` cap (#988) — the in-turn spend brake. A teammate - // with no declared budget gets no hook, which matches the vendored - // runtime's own posture: openhuman constructs `BudgetStopHook` nowhere - // and explicitly "never hard-stops a user-present turn that isn't - // actively burning a live budget". A turn that never outruns a real - // budget has nothing to protect it from, and a blanket magic number no - // operator can see or change would be worse than none. - // - // A budget halt and an iteration-cap pause are **different outcomes**, not - // two spellings of one: openhuman reports the cap through - // `Agent::last_turn_hit_cap`, which stays `false` for a hook-driven stop - // (the run paused below `max_tool_iterations`, so its cap predicate does - // not hold). Part 1 of #926 makes the cap pause operator-visible; it must - // not inherit budget halts. - let mut hooks: Vec> = Vec::new(); - if let Some(control) = steer { - hooks.push(Arc::new(crate::harness::steer::SteerStopHook::new( - control.clone(), - ))); - } - // Issue #1032: the budget hook is *wrapped* rather than pushed bare, so - // the halt survives the boundary. Upstream's `StopDecision::Stop` is - // consumed inside openhuman's tool loop, which returns the run's text as - // an ordinary `Ok(reply)`; `with_stop_hooks` hands back only the - // future's value; and `last_turn_hit_cap()` is `false` here by design. - // Without the wrapper there is nothing left to read, and a turn stopped - // for spend is indistinguishable from one that finished. - // - // The predicate itself stays upstream's — the wrapper only observes it. - let mut spend_brake: Option<(f64, Arc)> = None; - if let Some(cap) = self.turn_spend_cap_usd() { - let hook = crate::harness::spend::SpendStopHook::new(cap); - // Taken before the hook is boxed into the task-local list; once it - // is an `Arc` the concrete type is unreachable. - spend_brake = Some((cap, hook.halted())); - hooks.push(Arc::new(hook)); - } - - // `Box::pin` at the task-local scope boundary (the nested-scope - // stack-overflow trap). The turn body owns the retry classification and - // reports every attempt's usage. - let (reply, usages): (crate::Result, Vec) = - oh::agent::stop_hooks::with_stop_hooks( - hooks, - Box::pin(async { - let mut usages: Vec = Vec::new(); - let first = agent.turn(message).await; - usages.push(read_turn_usage(&agent)); - let reply: crate::Result = match self.classify_turn(first) { - AttemptOutcome::Reply(reply) => Ok(reply), - AttemptOutcome::Hard(err) => Err(err), - AttemptOutcome::Empty => { - // Retry-guard edge: skip the one-shot retry when an - // operator steer already pends, so a cancel/pause - // before any text can't restart the work. - // - // Issue #1032 adds the second guard, on the same - // reasoning: the work was stopped on purpose, and an - // empty reply is not licence to restart it. The - // retry is a fresh `agent.turn`, so openhuman builds - // it a fresh `TurnCost` — the brake's accumulator - // starts back at zero, and a teammate that had just - // exhausted its cap could spend up to a whole cap - // again before the hook fired a second time. The - // brake is armed per turn, so nothing else here - // would stop it. - // - // **Defence in depth, not a fix to an observed bug, - // and the difference is recorded so nobody re-derives - // it.** The `Empty` arm appears to be unreachable - // after a halt: a halt implies at least one completed - // tool iteration, and openhuman answers the post-halt - // wrap-up with its own synthesised "here's what I did - // this turn" summary — which it substitutes even when - // the wrap-up call returns blank text OR no choices - // at all. Both were scripted against the real turn - // loop and neither reached this arm, so there is no - // test here that would fail without this guard, and - // one was deliberately not left behind pretending - // otherwise. What the guard buys is that the - // invariant stops depending on that substitution - // staying true across a vendored bump. - // - // `halted_for_spend` below still reports the halt - // either way, so the operator gets the notice that - // explains a stub reply rather than silence. - let spend_halted = spend_brake.as_ref().is_some_and(|(_, halted)| { - halted.load(std::sync::atomic::Ordering::SeqCst) - }); - if steer.map(|c| c.requested()).unwrap_or(false) || spend_halted { - Ok(crate::harness::mcp_probe::scrub(GRACEFUL_EMPTY_REPLY, &[])) - } else { - let second = agent.turn(message).await; - usages.push(read_turn_usage(&agent)); - match self.classify_turn(second) { - AttemptOutcome::Reply(reply) => Ok(reply), - AttemptOutcome::Empty => Ok(crate::harness::mcp_probe::scrub( - GRACEFUL_EMPTY_REPLY, - &[], - )), - AttemptOutcome::Hard(err) => Err(err), - } - } - } - }; - (reply, usages) - }), - ) - .await; - - // Detach the sink (drops the only remaining `Sender`, closing the - // channel), release the agent lock, then drain + fold. A `Hard` error - // still runs this cleanup before propagating, so the collector never - // leaks. - agent.set_on_progress(None); - // Issue #926: read the cap flag while the lock is still held, the same - // under-lock idiom `read_turn_usage` uses above. Not draining, so the - // retry path's second attempt simply overwrites the first's value — - // which is right: the outcome describes the attempt that produced the - // reply being returned. - let hit_iteration_cap = agent.last_turn_hit_cap(); - drop(agent); - let events = collector.await.unwrap_or_default(); - // The cap openhuman was actually enforcing, for the trace only. Taken - // from the last `IterationStarted` rather than from config, so the log - // reports the number the turn ran under instead of the one this crate - // believes it configured. Deliberately NOT plumbed into the operator - // notice: one notice can cover a responder turn, a desk turn and a - // relay turn, and naming one of their caps would be a number the - // operator cannot map back to anything. - let iteration_cap = events.iter().rev().find_map(|event| match event { - oh::agent::progress::AgentProgress::IterationStarted { max_iterations, .. } => { - Some(*max_iterations) - } - _ => None, - }); - if hit_iteration_cap { - tracing::info!( - agent = %self.agent_id, - iteration_cap, - "[turn] paused at the tool-iteration cap; the reply is a resumable checkpoint, not a finished answer" - ); - } - // Issue #1032: read the spend brake the same way. Not under the agent - // lock — the flag lives on the hook, not on the vendored session, and - // the hook has already finished running by the time `with_stop_hooks` - // returns. - // - // The spend is summed over every attempt's usage rather than read from - // the hook, so the figure covers the retry path's second attempt too: - // both were paid for, and reporting only one would understate what the - // turn actually cost. - let halted_for_spend = spend_brake.and_then(|(cap_usd, halted)| { - halted - .load(std::sync::atomic::Ordering::SeqCst) - .then(|| SpendHalt { - agent: self.agent_id.clone(), - spent_usd: usages.iter().map(|usage| usage.cost_usd).sum(), - cap_usd, - }) - }); - if let Some(halt) = &halted_for_spend { - tracing::info!( - agent = %self.agent_id, - spent_usd = halt.spent_usd, - cap_usd = halt.cap_usd, - "[turn] halted at the in-turn spend cap; the reply stops short of the work it was doing" - ); - } - let steps = steps::fold_steps(events); - - let reply = reply?; - Ok(( - TurnOutcome { - reply, - steps, - hit_iteration_cap, - halted_for_spend, - }, - usages, - )) - } - - /// This turn's in-turn spend ceiling, in USD — the value that - /// [`BudgetStopHook`](oh::agent::stop_hooks::BudgetStopHook) halts the turn - /// at, armed only when the teammate declares a `budget_usd_daily` cap - /// (issue #988). `None` means no hook is installed. - /// - /// This mirrors the vendored runtime's own posture. OpenCompany's plan-level - /// token ceiling and a teammate's `budget_usd_daily` are **pre-dispatch** — - /// they decide whether to start a turn and cannot see inside one — and - /// openhuman itself constructs `BudgetStopHook` nowhere, applying only an - /// opt-in token-based goal hook. So this crate, like upstream, arms the - /// in-turn brake only for a teammate who has opted into a budget: a declared - /// `budget_usd_daily` cap also bounds any single turn of that teammate's, so - /// the worst-case overshoot is "one daily cap" rather than "one turn, of - /// unknown size". A teammate with no declared budget gets no hook — the - /// runtime never hard-stops a turn that isn't actively burning a live budget - /// — and there is no blanket magic number no operator can see or change. - /// - /// A non-finite or non-positive manifest value is ignored (no hook armed) - /// rather than forwarded: the vendored hook fails closed on a malformed cap - /// and would halt every turn at iteration one. Such a teammate is already - /// refused before dispatch (`spent >= cap` holds at zero spend), so this only - /// guards the path where no meter was available to make that call. - fn turn_spend_cap_usd(&self) -> Option { - match self.budget_usd_daily { - Some(daily) if daily.is_finite() && daily > 0.0 => Some(daily), - _ => None, - } - } - - /// Classify one `agent.turn` result for the retry wrapper. - fn classify_turn(&self, result: anyhow::Result) -> AttemptOutcome { - match result { - Ok(reply) if reply.trim().is_empty() => AttemptOutcome::Empty, - Ok(reply) => AttemptOutcome::Reply(reply), - Err(err) if is_transient_empty_response(&err) => AttemptOutcome::Empty, - Err(err) => AttemptOutcome::Hard(OpenCompanyError::Harness(format!( - "turn for '{}': {err}", - self.agent_id - ))), - } - } -} - -/// Reads the just-completed turn's usage (zero when the provider reported none). -fn read_turn_usage(agent: &Agent) -> TurnUsage { - agent - .last_turn_usage() - .map(|u| TurnUsage { - input_tokens: u.input_tokens, - output_tokens: u.output_tokens, - cached_input_tokens: u.cached_input_tokens, - cost_usd: u.cost_usd, - }) - .unwrap_or_default() -} - -/// Whether a turn error is the transient empty-response class openhuman raises -/// instead of a silent blank reply. Matched on the error chain's message -/// (`turn` returns `anyhow::Result`, so the typed `AgentError` is erased): -/// "The model returned an empty response…". -fn is_transient_empty_response(err: &anyhow::Error) -> bool { - format!("{err:#}") - .to_ascii_lowercase() - .contains("empty response") -} - -/// What a workspace-ensure attempt should say, given what the last attempt for -/// the same agent said (issue #449). -/// -/// The attempt itself is per dispatch and stays that way — see -/// [`note_workspace_attempt`](HarnessPool::note_workspace_attempt) for why -/// memoising it is the wrong fix. Only the *reporting* is edge-triggered. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum WorkspaceReport { - /// The first failure since this agent was last healthy: report it. - Failed, - /// Still failing, and already reported: say nothing. - StillFailing, - /// Working again after a reported failure: say so once, so a reader who saw - /// the error learns it ended. - Recovered, - /// Working, and was already working: say nothing. - StillHealthy, -} - -impl WorkspaceReport { - /// Whether this transition has anything to log at all. - pub(crate) fn is_silent(self) -> bool { - matches!(self, Self::StillFailing | Self::StillHealthy) - } -} - -/// Folds one attempt's outcome into the set of currently-failing keys and -/// returns what to report. -/// -/// Pure but for the `failing` set it edits, so the whole state machine is -/// testable without a model, a roster or a filesystem. `failing` holds exactly -/// the keys whose last attempt failed **and** whose failure has been reported; -/// `failed` is this attempt's outcome. -fn workspace_report(failing: &mut HashSet, key: &K, failed: bool) -> WorkspaceReport -where - K: std::hash::Hash + Eq + Clone, -{ - if failed { - // `insert` returns false when the key was already there — i.e. the - // previous attempt failed and was already reported. - if failing.insert(key.clone()) { - WorkspaceReport::Failed - } else { - WorkspaceReport::StillFailing - } - } else if failing.remove(key) { - WorkspaceReport::Recovered - } else { - WorkspaceReport::StillHealthy - } -} - -/// A pool of live agents, one roster per company. -pub struct HarnessPool { - agents: RwLock>>>, - /// Fingerprint of the effective MCP server set the cached roster was built - /// from, keyed by company. Drives MCP-freshness: [`ensure`](Self::ensure) - /// rebuilds the roster whenever the fingerprint changes. - mcp_fingerprints: RwLock>, - /// Fingerprint of the overlay-agent set (issue #71 — Active Runtime - /// Teammates) the cached roster was built from, keyed by company. Drives - /// overlay-agent freshness: [`ensure`](Self::ensure) rebuilds the roster - /// whenever an operator- or orchestrator-added teammate is added/removed, - /// mirroring the MCP-freshness fingerprint above. - overlay_fingerprints: RwLock>, - /// Fingerprint of the resolved [`CapabilityFilter`](toolbelt::CapabilityFilter) - /// the cached roster was built from, keyed by company (issue #108). Drives - /// capability-budget freshness: [`ensure`](Self::ensure) re-resolves the - /// tenant's filter from the [`UsageMeter`] on every call and rebuilds the - /// roster whenever the denied-namespace set changes — so a tier that crosses - /// its token budget switches off on the company's **next** turn. With no - /// plan ([`HarnessDeps::plan`] `None`) the filter is the static - /// [`HarnessDeps::capabilities`], whose fingerprint never moves — no rebuild, - /// byte-identical to Cell A. - capability_fingerprints: RwLock>, - /// Fingerprint of the resolved per-tenant [`TenantComposio`](composio::TenantComposio) - /// config the cached roster was built from, keyed by company (issue #110). - /// Drives Composio-freshness: [`ensure`](Self::ensure) re-resolves the token - /// (+ toolkit allowlist) from the [`SecretStore`] on every call and rebuilds - /// the roster whenever it changes — so a console token set/rotate/clear takes - /// effect on the company's **next** turn with no restart. With no secret - /// store wired the config is the static [`HarnessDeps::composio`], whose - /// fingerprint never moves. - composio_fingerprints: RwLock>, - /// Fingerprint of the billing connections (Chargebee #788, PayPal #789) the - /// cached roster was built from, keyed by company. - /// - /// Without this axis a credential saved from the console reaches nothing - /// until a restart — the roster is cached, so `build_agent` is never called - /// again to notice it. That was live for both integrations until the tools - /// were observed missing from an agent whose settings page said "Connected". - billing_fingerprints: RwLock>, - /// Fingerprint of the company's bound-repository set the cached roster was - /// built from, keyed by company (issue #245). Drives repository freshness: - /// [`ensure`](Self::ensure) re-reads the binding index from the - /// [`SecretStore`] on every call and rebuilds the roster whenever it moves — - /// so a bind, a credential rotation and a revoke each reach the agent on the - /// company's **next** turn with no restart. - /// - /// All three have to move it, which is why the fingerprint is over - /// `(key, token_fingerprint, branches)` rather than over the key alone: a - /// rotation changes nothing about *which* repositories exist, and a roster - /// that kept a tool description naming a binding whose credential has since - /// been revoked would offer an agent a checkout that can no longer fetch. - /// With no secret store wired the set is the static - /// [`HarnessDeps::repo_bindings`], whose fingerprint never moves. - repo_fingerprints: RwLock>, - /// Fingerprint of the operator skill-delta set the cached roster was built - /// from, keyed by company (issue #41). Drives skill-delta freshness: - /// [`ensure`](Self::ensure) re-fetches the deltas from the - /// [`SkillStateStore`](crate::ports::skills_state::SkillStateStore) on every - /// call and rebuilds the roster whenever they change — so a skill - /// authored / edited / enabled / disabled in the console Skills tab reaches - /// the agent on the company's **next** turn with no restart. Without this - /// axis the four fingerprints above are all stable on a skills-only change, - /// the fast path returns early, and the new skill never surfaces until a - /// process restart (the regression this fixes). With no skill store wired - /// the delta set is always empty — stable fingerprint, no rebuild. - skill_fingerprints: RwLock>, - /// Fingerprint of the operator budget-override set the cached roster was - /// built from, keyed by company (issue #343). Drives budget freshness: - /// [`ensure`](Self::ensure) re-resolves the overrides from - /// [`HarnessDeps::store`] on every call and rebuilds the roster whenever a - /// cap is set, changed, cleared or reset — so a budget edited on the console - /// Team page reaches the dispatch gate and the per-agent - /// [`ApprovalPolicy`](policy::ApprovalPolicy) on the company's **next** turn, - /// with no restart and no redeploy. That is the entire point of #343: the - /// cap is enforced from the roster, and without this axis every other - /// fingerprint is stable on a budget-only change, so the fast path would - /// reuse a roster still carrying the old cap until the process restarted. - /// A company that never sets an override keeps an empty set and a stable - /// fingerprint — no rebuild, byte-identical to the pre-#343 behaviour. - budget_fingerprints: RwLock>, - /// Per-company fingerprint of the operator `[policy]` override (issue #562), - /// so a console tier change rebuilds the roster instead of waiting for a - /// restart. Without this axis the override persists and is silently ignored: - /// `ApprovalPolicy` is built once per roster, not once per call. - policy_fingerprints: RwLock>, - /// Per-company fingerprint of the desk scoping a roster's grants resolve - /// through — which desks exist, who sits on them, and each one's tool - /// ceiling. - /// - /// Needed for the same reason as [`Self::budget_fingerprints`]: a tool belt - /// is wired once per roster, not once per call, so without this axis a - /// console desk-ceiling edit (or seating a teammate on a restricted desk) - /// would leave every other fingerprint stable and the fast path would keep - /// serving the old belt until the process restarted. A company whose desks - /// declare no ceilings keeps a stable fingerprint and never rebuilds on this - /// axis. - desk_fingerprints: RwLock>, - /// Per-company fingerprint of the routed workspace documents — hashed over - /// their **bodies**, not merely their names. - /// - /// A persona is assembled once per roster, so without this axis an operator - /// editing a routed note would leave every other fingerprint stable and the - /// fast path would keep serving a prompt quoting the old text until the - /// process restarted. Hashing the names alone would have exactly that bug, - /// since the routing table does not move when a document's contents do — - /// which is the whole reason the routing layer is worth having. - /// - /// A company with no workspace store wired, or whose roles route nothing, - /// keeps a stable fingerprint and never rebuilds on this axis. - context_fingerprints: RwLock>, - /// The `(company, agent)` pairs whose last workspace-ensure failed and whose - /// failure has already been reported (issue #449). - /// - /// Not a memo of the *attempt* — see - /// [`note_workspace_attempt`](Self::note_workspace_attempt). Purely a record - /// of what has already been said, so an unmountable volume produces one - /// error line instead of one per turn forever. - /// - /// A `std::sync::Mutex` rather than a `tokio::sync::RwLock` like its - /// neighbours: the critical section is a single hash lookup with no `await` - /// in it, so the async lock would buy nothing and cost a scheduling point on - /// the dispatch path. - workspace_failures: std::sync::Mutex>, -} - -impl Default for HarnessPool { - fn default() -> Self { - Self::new() - } -} - -/// Whether a turn tees its progress onto the live [`turn_stream`](crate::turn_stream) -/// bus, and if so which chat thread its frames route to. `Off` for a turn with no -/// operator chat bubble (a dispatched task card or workflow agent node) — those -/// frames would misattribute to whatever thread most recently sent, so they -/// publish nothing (#125 review). `On { chat_id }` streams; `chat_id` is the -/// thread the durable reply is journaled under (`AgentReply.chat_id`), falling -/// back to the default desk when the caller addressed none. -#[derive(Clone, Copy)] -enum LiveStream<'a> { - Off, - On { chat_id: Option<&'a str> }, -} - -impl HarnessPool { - /// Builds an empty pool. - pub fn new() -> Self { - Self { - agents: RwLock::new(HashMap::new()), - mcp_fingerprints: RwLock::new(HashMap::new()), - overlay_fingerprints: RwLock::new(HashMap::new()), - capability_fingerprints: RwLock::new(HashMap::new()), - composio_fingerprints: RwLock::new(HashMap::new()), - billing_fingerprints: RwLock::new(HashMap::new()), - repo_fingerprints: RwLock::new(HashMap::new()), - skill_fingerprints: RwLock::new(HashMap::new()), - budget_fingerprints: RwLock::new(HashMap::new()), - policy_fingerprints: RwLock::new(HashMap::new()), - desk_fingerprints: RwLock::new(HashMap::new()), - context_fingerprints: RwLock::new(HashMap::new()), - workspace_failures: std::sync::Mutex::new(HashSet::new()), - } - } - - /// Records one workspace-ensure outcome for `(company, agent)` and returns - /// what it should say. - /// - /// **The attempt stays per dispatch.** The obvious fix for a repeating log - /// line — remember that this agent's workspace was already handled and stop - /// trying — is the wrong one in both directions, and this is why the - /// suppression is on the reporting rather than on the work: - /// - /// * Memoising **success** means a data dir wiped or restored *after* the - /// first successful turn is never noticed again, and every relative file - /// write is refused for the life of the process — the exact regression - /// issue #409 added the per-dispatch retry to prevent. - /// * Memoising **failure** means a volume that mounts a second late never - /// recovers, because nothing ever tries again. - /// - /// Both trade a noisy log for a broken agent. The retry is cheap (two - /// syscalls on the already-exists path, against a turn about to call a - /// model) and it is what makes the condition self-healing, so it keeps - /// running every time. What changes is that a persistent failure is stated - /// once rather than once per turn. - fn note_workspace_attempt( - &self, - company: &CompanyId, - agent_id: &str, - failed: bool, - ) -> WorkspaceReport { - let key = (company.clone(), agent_id.to_string()); - let mut failing = self - .workspace_failures - .lock() - .expect("workspace-failure set poisoned"); - workspace_report(&mut failing, &key, failed) - } - - /// Ensures a company's roster is built and cached. - /// - /// **MCP-freshness (the error-hardening cell)**: on every call, the effective - /// MCP server set is re-resolved (from the [`SecretStore`] when - /// [`HarnessDeps::secrets`] is wired, else the boot-resolved - /// [`HarnessDeps::mcp_servers`]) and fingerprinted. The roster is rebuilt when - /// it is absent **or** the fingerprint changed — so a console MCP - /// add/remove/enable-toggle reaches the agent on its **next turn**, with no - /// company restart (the "Parallel Search / BrowserBase" bug). When nothing - /// changed, the cached roster is reused (the common fast path), exactly as - /// before. - /// - /// **Overlay-agent freshness (issue #71)**: the live overlay-agent set is - /// re-resolved and fingerprinted the same way, from [`HarnessDeps::store`] - /// rather than the (possibly stale) `company` snapshot passed in — so a - /// teammate added through the console `POST .../team` route or the - /// orchestrator's `add_agent` tool becomes a real, addressable roster agent - /// on the company's **next** `ensure` call, with no restart. - /// - /// **Skill-delta freshness (issue #41)**: the operator skill deltas are - /// fetched from [`HarnessDeps::skills`] and fingerprinted **before** the - /// fast-path staleness check (not after it, as they were — the regression), - /// so a skill authored / edited / enabled / disabled in the console Skills - /// tab rebuilds the roster and reaches the agent on its **next** turn, even - /// when every other axis (MCP, overlay, capability, composio) is unchanged. - /// With no skill store wired the delta set is empty and the fingerprint is - /// stable — no rebuild, exactly as before. - /// - /// **Budget freshness (issue #343)**: the operator's per-teammate daily - /// spend caps ride the same live [`HarnessDeps::store`] read as the overlay - /// agents and are fingerprinted alongside them, so a cap set, raised, - /// cleared or reset from the console Team page rebuilds the roster and is - /// enforced on the company's **next** dispatch. Nothing downstream had to - /// change for this: the L1 gate in [`Self::run`] reads - /// [`CompanyAgent::budget_usd_daily`] and the policy arm reads the - /// [`ApprovalPolicy`](policy::ApprovalPolicy) both roster-built here, so - /// rebuilding the roster *is* the enforcement update. That is what makes - /// "no restart, no redeploy" a property of the design rather than a claim. - pub async fn ensure(&self, company: &CompanyRecord, deps: &HarnessDeps) -> crate::Result<()> { - // Re-resolve + fingerprint the effective MCP set (cheap; no rebuild yet). - let effective_mcp = self.resolve_effective_mcp(company, deps).await; - let mcp_fp = mcp_fingerprint(&effective_mcp); - - // Re-resolve + fingerprint the live overlay-agent set the same way, and - // the operator budget overrides riding the same store read (issue #343). - let overlay = self.resolve_effective_overlay(company, deps).await; - let overlay_fp = overlay_fingerprint(&overlay.agents); - let budget_fp = budget_fingerprint(&overlay.budgets); - let policy_fp = policy_fingerprint(overlay.policy.as_ref()); - // Desk scoping now decides capability (the middle level of the - // three-level narrowing), so it joins the staleness check: without this - // a console desk-ceiling edit — or seating a teammate on a restricted - // desk — would not reach the roster until a restart. - let desk_fp = - desk_scope_fingerprint(&overlay.desks, &overlay.desk_members, &overlay.desk_tools); - - // Re-resolve + fingerprint the tenant's capability filter (issue #108): - // a per-tenant, per-period, fail-closed budget read from the meter. With - // no plan this is the static `deps.capabilities`, whose fingerprint is - // stable — so a no-plan company never rebuilds on this axis. - let capability_filter = self.resolve_capability_filter(company, deps).await; - let capability_fp = capability_budget::filter_fingerprint(&capability_filter); - - // Re-resolve + fingerprint the per-tenant Composio config (issue #110): - // the token (+ toolkit allowlist) read live from the secret store, so a - // console token set/rotate/clear takes effect on the next turn. With no - // secret store wired this is the static `deps.composio`, whose - // fingerprint is stable — so that company never rebuilds on this axis. - let composio_config = self.resolve_composio(company, deps).await; - let composio_fp = composio::TenantComposio::fingerprint(&composio_config); - - // Re-resolve + fingerprint the billing connections (#788, #789) for the - // same reason as Composio above: both are set from the console, so a - // roster that never re-reads them leaves an agent without billing tools - // on a company whose settings page reads "Connected". - #[cfg(feature = "chargebee")] - let chargebee_config = self.resolve_chargebee(company, deps).await; - #[cfg(feature = "paypal")] - let paypal_config = self.resolve_paypal(company, deps).await; - // The hosting credential is set from the same settings surface and goes - // stale the same way, so it rides the same axis. - let hosting_config = self.resolve_hosting(company, deps).await; - // A build without either feature has no billing axis to go stale on, so - // the fingerprint is a constant and this company never rebuilds on it. - let billing_fp = { - use std::hash::Hasher; - // Always written to: the hosting axis below is ungated. - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - #[cfg(feature = "chargebee")] - hasher.write_u64(chargebee::TenantChargebee::fingerprint(&chargebee_config)); - #[cfg(feature = "paypal")] - hasher.write_u64(paypal::TenantPaypal::fingerprint(&paypal_config)); - hasher.write_u64(hosting::TenantHosting::fingerprint(&hosting_config)); - hasher.finish() - }; - - // Re-read + fingerprint the company's bound repositories (issue #245): - // one index document, read live, so a bind / rotate / revoke reaches the - // agent on the next turn. Only companies that explicitly grant `repo` - // touch the store on this axis; everything else resolves to the static - // `deps.repo_bindings` (empty at every construction site but the - // production builder), whose fingerprint never moves. - let repo_bindings = self.resolve_repo_bindings(company, deps).await; - let repo_fp = repo_binding_fingerprint(&repo_bindings); - - // Re-fetch + fingerprint the operator skill deltas (issue #41) BEFORE the - // fast-path check. A skills-only change leaves every other axis stable, so - // unless skills participate in the staleness check the cached roster is - // wrongly reused and a console-authored / edited / disabled skill never - // surfaces until a restart (the regression). `build_roster`/`build_agent` - // stay synchronous and fold these deltas into each agent's effective - // skill set; the same Vec is reused for the rebuild below (no re-fetch). - let mut skill_deltas = match &deps.skills { - Some(store) => store.list(&company.id).await?, - None => Vec::new(), - }; - // `[globals].disable = ["skill:…"]` reaches the effective set as a - // synthesized disabling delta rather than a second opt-out mechanism - // inside `EffectiveSkills`: the manifest and the console are then saying - // the same thing in the same vocabulary, and a disable always beats an - // enable there, so the company's own declaration wins over a console - // re-enable of a skill it opted out of. - skill_deltas.extend(globals_skill_disables(&company.manifest.globals.disable)); - let skill_deltas = skill_deltas; - let skill_fp = skill_delta_fingerprint(&skill_deltas); - - // Resolve the routed workspace documents (context routing) before the - // fast-path check, and fingerprint their *content*. Both halves matter: - // resolving here is what lets the synchronous `build_agent` fold them - // into a persona at all, and hashing the bodies rather than the file - // names is what makes an operator's edit to a routed note rebuild the - // roster. A name-only hash would leave an edited note invisible until a - // restart — the same staleness bug `skill_fp` above exists to close. - let routed_context = self - .resolve_routed_context(company, deps, &overlay.agents) - .await; - let context_fp = routed_context_fingerprint(&routed_context); - - { - let agents = self.agents.read().await; - let mcp_fingerprints = self.mcp_fingerprints.read().await; - let overlay_fingerprints = self.overlay_fingerprints.read().await; - let capability_fingerprints = self.capability_fingerprints.read().await; - let composio_fingerprints = self.composio_fingerprints.read().await; - let billing_fingerprints = self.billing_fingerprints.read().await; - let repo_fingerprints = self.repo_fingerprints.read().await; - let skill_fingerprints = self.skill_fingerprints.read().await; - let budget_fingerprints = self.budget_fingerprints.read().await; - let policy_fingerprints = self.policy_fingerprints.read().await; - let desk_fingerprints = self.desk_fingerprints.read().await; - let context_fingerprints = self.context_fingerprints.read().await; - if agents.contains_key(&company.id) - && mcp_fingerprints.get(&company.id) == Some(&mcp_fp) - && overlay_fingerprints.get(&company.id) == Some(&overlay_fp) - && capability_fingerprints.get(&company.id) == Some(&capability_fp) - && composio_fingerprints.get(&company.id) == Some(&composio_fp) - && billing_fingerprints.get(&company.id) == Some(&billing_fp) - && repo_fingerprints.get(&company.id) == Some(&repo_fp) - && skill_fingerprints.get(&company.id) == Some(&skill_fp) - && budget_fingerprints.get(&company.id) == Some(&budget_fp) - && policy_fingerprints.get(&company.id) == Some(&policy_fp) - && desk_fingerprints.get(&company.id) == Some(&desk_fp) - && context_fingerprints.get(&company.id) == Some(&context_fp) - { - return Ok(()); - } - } - - // Fold the freshly-resolved MCP set into the deps the roster is built - // from, so a changed set actually reaches the rebuilt agents. The clone - // shares every Arc / queue handle — only `mcp_servers` is overridden. - let mut fresh_deps = deps.clone(); - fresh_deps.mcp_servers = effective_mcp; - // Install the freshly-resolved capability filter on the deps the roster - // is built from, the same pattern as `mcp_servers` — so a tenant that - // crossed a tier budget gets a roster whose exec tools are actually - // trimmed. With no plan this is just `deps.capabilities` unchanged. - fresh_deps.capabilities = capability_filter; - // Install the freshly-resolved Composio config the same way, so a token - // set/rotate/clear reaches the rebuilt agents (issue #110). - fresh_deps.composio = composio_config; - #[cfg(feature = "chargebee")] - { - fresh_deps.chargebee = chargebee_config; - } - #[cfg(feature = "paypal")] - { - fresh_deps.paypal = paypal_config; - } - fresh_deps.hosting = hosting_config; - // And the freshly-read bindings (issue #245), so a repository bound or - // revoked in the console is what the rebuilt agents' tools resolve - // against — including the descriptions that name what is bound. - fresh_deps.repo_bindings = repo_bindings; - // Same treatment for the overlay-agent set: `company` may be a stale - // boot-time snapshot (e.g. `HarnessBrain::record`), so the roster is - // built from the live-resolved overlay set, not `company.overlay_agents`. - let mut fresh_company = company.clone(); - fresh_company.overlay_agents = overlay.agents; - // Same treatment for the budget overrides (issue #343): `build_roster` - // resolves every agent's cap through `fresh_company.effective_budget`, - // so installing the live set here is what carries a console budget edit - // into the roster the very next turn runs on. - fresh_company.overlay_budgets = overlay.budgets; - // The desk axis gets the same treatment, and needs it for the same - // reason: `build_roster` resolves every teammate's grants through - // `fresh_company.agent_desk_tools`, so the live desk set, seating and - // ceilings have to be the ones installed here. - fresh_company.overlay_desks = overlay.desks; - fresh_company.overlay_desk_members = overlay.desk_members; - fresh_company.overlay_desk_tools = overlay.desk_tools; - // Issue #562: same treatment for the policy override — `build_roster` - // resolves the tier through `fresh_company.effective_policy`, so installing - // the live value here is what carries a console tier change into the roster - // the next turn runs on. - fresh_company.overlay_policy = overlay.policy; - - // Issue #551 note — this rebuild deliberately touches no workspace. - // - // It used to provision `Agents//` for the roster it was about to - // build, because a teammate added at runtime (a manifest edit, the - // console's `add_member`, the orchestrator's `add_agent`) all land here - // as a moved overlay fingerprint and boot could not have known about - // them. That justification is gone: a member folder is no longer a - // function of the roster. `Agents/` and `Desks/` are laid down once at - // boot ([`RuntimeBuilder::build`]) and depend on nothing a rebuild can - // change, and `Agents//` is minted by - // [`ensure_agent_folder`](crate::company::workspace_scaffold::ensure_agent_folder) - // at the moment that agent first produces something — which is also the - // repair path if boot's create ever fail-softed, since the minter - // creates the root it needs. A rebuild-time call would now be a tree - // read that can only ever find its work already done. - let roster = build_roster(&fresh_company, &fresh_deps, &skill_deltas, &routed_context)?; - - let mut agents = self.agents.write().await; - agents.insert(company.id.clone(), roster); - self.mcp_fingerprints - .write() - .await - .insert(company.id.clone(), mcp_fp); - self.overlay_fingerprints - .write() - .await - .insert(company.id.clone(), overlay_fp); - self.capability_fingerprints - .write() - .await - .insert(company.id.clone(), capability_fp); - self.composio_fingerprints - .write() - .await - .insert(company.id.clone(), composio_fp); - self.billing_fingerprints - .write() - .await - .insert(company.id.clone(), billing_fp); - self.repo_fingerprints - .write() - .await - .insert(company.id.clone(), repo_fp); - self.skill_fingerprints - .write() - .await - .insert(company.id.clone(), skill_fp); - self.budget_fingerprints - .write() - .await - .insert(company.id.clone(), budget_fp); - self.policy_fingerprints - .write() - .await - .insert(company.id.clone(), policy_fp); - self.desk_fingerprints - .write() - .await - .insert(company.id.clone(), desk_fp); - self.context_fingerprints - .write() - .await - .insert(company.id.clone(), context_fp); - Ok(()) - } - - /// Re-resolves the company's capability filter (issue #108): with a plan - /// wired ([`HarnessDeps::plan`]), a per-tenant, per-period, fail-closed - /// budget read from the [`UsageMeter`] via - /// [`capability_budget::resolve_filter`]; without one, the static - /// [`HarnessDeps::capabilities`] verbatim (gating off). Never a boot - /// snapshot — resolved on every `ensure` so a tier switches off the turn - /// after its budget is crossed. - async fn resolve_capability_filter( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - ) -> toolbelt::CapabilityFilter { - match &deps.plan { - Some(plan) => { - capability_budget::resolve_filter( - plan, - deps.meter.as_deref(), - &company.id, - crate::ports::now_millis(), - ) - .await - } - None => deps.capabilities.clone(), - } - } - - /// Re-resolves the company's per-tenant Composio config (issue #110) from the - /// [`SecretStore`], so a console token set/rotate/clear takes effect on the - /// next turn. Only companies that **explicitly** grant `composio` touch the - /// secret store on this axis; others resolve to `None` (no tools). With no - /// secret store wired this degrades to the static [`HarnessDeps::composio`]. - /// - /// Resolution prefers the company's own stored token and falls back to this - /// instance's platform identity; with neither it yields `None` (fail closed). - /// Both the backend URL (from [`composio::COMPOSIO_BACKEND_URL_ENV`], then the - /// tenant API base [`composio::TINYHUMANS_API_URL_ENV`], then the prod - /// default) and the platform identity are read process-globally here, so a - /// live re-resolution keeps them even when nothing was stored at boot. - /// - /// Re-deriving the token source every turn costs nothing — building it reads - /// no file — and the roster that keeps it holds one instance for its whole - /// lifetime, so its rotation cache still works. - async fn resolve_composio( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - ) -> Option { - if !crate::company::grants_composio_explicit(&company.manifest.tools.allow) { - return None; - } - let toolkits = company.manifest.tools.composio.toolkits.clone(); - match &deps.secrets { - Some(secrets) => { - use crate::app::config::EnvSource; - let env = crate::app::config::ProcessEnv; - let url = env.get(composio::COMPOSIO_BACKEND_URL_ENV); - let api_url = env.get(composio::TINYHUMANS_API_URL_ENV); - composio::TenantComposio::resolve( - &company.id, - secrets.as_ref(), - toolkits, - url, - api_url, - crate::company::TinyhumansTokenSource::from_env(&env).map(std::sync::Arc::new), - ) - .await - } - None => deps.composio.clone(), - } - } - - /// Re-reads the company's Chargebee connection from the secret store, so a - /// key saved or rotated in Settings → Billing reaches the agent on its next - /// turn rather than at the next restart (issue #788). - /// - /// Only companies that **explicitly** grant `chargebee` read at all. With no - /// secret store wired this keeps the boot-resolved - /// [`HarnessDeps::chargebee`] — which was itself resolved from *this* - /// company's secret store by the runtime builder, so the fallback cannot - /// reach another tenant's credential. - /// - /// A transient **read error** keeps that connection too, with a warning, - /// rather than un-wiring the billing tools — the same direction - /// [`Self::resolve_repo_bindings`] and [`Self::resolve_effective_mcp`] - /// degrade in, and the safe one here for a specific reason: a stale - /// Chargebee credential is refused by Chargebee, which the agent surfaces as - /// a tool error it can report, whereas a tool that has vanished is invisible - /// to the agent — it simply stops being able to invoice and says nothing. - /// An absent credential still resolves to `None`; only the error case holds. - #[cfg(feature = "chargebee")] - async fn resolve_chargebee( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - ) -> Option { - if !crate::company::grants_chargebee_explicit(&company.manifest.tools.allow) { - return None; - } - let Some(secrets) = &deps.secrets else { - return deps.chargebee.clone(); - }; - match chargebee::TenantChargebee::resolve(secrets, &company.id).await { - Ok(resolved) => resolved, - Err(err) => { - tracing::warn!( - company = %company.id, - "[chargebee] could not read the billing credential; keeping the last known \ - connection: {err}" - ); - deps.chargebee.clone() - } - } - } - - /// The hosting equivalent, for the same reasons. - /// - /// Only companies that **explicitly** grant `hosting` read at all: a - /// deployment publishes a company's files to the public internet and can - /// provision a database it is billed for, so the catch-all `*` does not - /// confer it. - /// - /// A transient read error keeps the last known connection with a warning, - /// like `chargebee` and for the same reason: a stale hosting key is refused - /// by the provider, which the agent surfaces as a tool error it can report, - /// whereas a tool that has vanished is invisible to the agent — it simply - /// stops being able to deploy and says nothing. - async fn resolve_hosting( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - ) -> Option { - if !crate::company::grants_hosting_explicit(&company.manifest.tools.allow) { - return None; - } - let Some(secrets) = &deps.secrets else { - return deps.hosting.clone(); - }; - match hosting::TenantHosting::resolve(secrets, &company.id).await { - Ok(resolved) => resolved, - Err(err) => { - tracing::warn!( - company = %company.id, - "[hosting] could not read the hosting credential; keeping the last known \ - connection: {err}" - ); - deps.hosting.clone() - } - } - } - - /// The PayPal equivalent (issue #789), for the same reasons. - #[cfg(feature = "paypal")] - async fn resolve_paypal( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - ) -> Option { - if !crate::company::grants_paypal_explicit(&company.manifest.tools.allow) { - return None; - } - let Some(secrets) = &deps.secrets else { - return deps.paypal.clone(); - }; - match paypal::TenantPaypal::resolve(secrets, &company.id).await { - Ok(resolved) => resolved, - Err(err) => { - tracing::warn!( - company = %company.id, - "[paypal] could not read the billing credential; keeping the last known \ - connection: {err}" - ); - deps.paypal.clone() - } - } - } - - /// Re-reads the company's bound repositories (issue #245) from the - /// [`RepoManager`](crate::runtime::RepoManager), so a bind, a credential - /// rotation or a revoke reaches the roster on the next turn. - /// - /// Only companies that **explicitly** grant `repo` read at all; everything - /// else answers empty without touching the store, mirroring - /// [`Self::resolve_composio`]. A transient read error degrades to the - /// boot-resolved [`HarnessDeps::repo_bindings`] with a warning rather than - /// dropping an agent's repository tools mid-session — the same direction - /// [`Self::resolve_effective_mcp`] degrades in, and the safe one: a stale - /// binding list still resolves against real bindings, while an empty one - /// un-wires the tools entirely. - async fn resolve_repo_bindings( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - ) -> Vec { - if !crate::company::grants_repo_explicit(&company.manifest.tools.allow) { - return Vec::new(); - } - let Some(repos) = deps.repos.as_ref() else { - return deps.repo_bindings.clone(); - }; - match repos.list().await { - Ok(bindings) => bindings, - Err(err) => { - tracing::warn!( - company = %company.id, - "[repo] could not read the repository bindings; keeping the last known set: {err}" - ); - deps.repo_bindings.clone() - } - } - } - - /// Re-resolves the company's effective MCP server set: from the secret store - /// when [`HarnessDeps::secrets`] is wired (picking up console changes), else - /// the boot-resolved [`HarnessDeps::mcp_servers`] unchanged. A resolution - /// error degrades to the boot-resolved set rather than dropping MCP tools. - async fn resolve_effective_mcp( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - ) -> Vec { - match &deps.secrets { - Some(secrets) => { - let mut decls = crate::company::mcp::resolve_effective( - &company.id, - &deps.default_mcp_servers, - &company.manifest.mcp_servers, - secrets.as_ref(), - ) - .await - .unwrap_or_else(|_| deps.mcp_servers.clone()); - // Refresh any near-expiry console-OAuth credential before the - // registry is built, so an agent never sends a stale bearer. - refresh_oauth_decls(&company.id, &mut decls, secrets.as_ref()).await; - decls - } - None => deps.mcp_servers.clone(), - } - } - - /// Re-resolves the company's live overlay-agent set (issue #71) **and** its - /// operator budget overrides (issue #343): reloads the [`CompanyRecord`] - /// from [`HarnessDeps::store`] so a teammate added through the console - /// `POST .../team` route or the orchestrator's `add_agent` tool, and a cap - /// written through `PUT .../team/{id}/budget`, both reach the roster on the - /// company's next `ensure` call — the same live-re-resolution pattern as - /// [`Self::resolve_effective_mcp`]. A missing record or a store error - /// degrades to the `company` snapshot passed in (never worse than the - /// pre-#71 always-static behaviour). - /// - /// The two collections share **one** store round-trip deliberately: they - /// come off the same record, and splitting them would double the per-turn - /// read for no gain. - /// Resolves every roster member's routed workspace documents, keyed by agent - /// id (`docs/spec/runtime/orchestration/context-routing.md`). - /// - /// Runs here, in the async caller, because `build_roster` is synchronous and - /// the [`WorkspaceStore`](crate::ports::WorkspaceStore) is not — the same - /// split as the skill deltas beside it. - /// - /// **Fails soft, per agent.** A store error yields no documents for that - /// role rather than failing the rebuild: routing enriches a prompt, and a - /// company whose workspace read hiccuped should answer from a thinner prompt - /// rather than stop answering. An unwired store (`None`) resolves to an - /// empty map, which is the pre-routing behaviour exactly. - /// - /// Overlay teammates are included: they are real roster agents that - /// [`build_roster`] builds the same way, so leaving them out would give a - /// console-added teammate a silently different prompt from a manifest one. - async fn resolve_routed_context( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - overlay_agents: &[OverlayAgent], - ) -> HashMap> { - let Some(workspace) = deps.workspace.as_ref() else { - return HashMap::new(); - }; - - // A manifest agent wins an id collision, exactly as `build_roster` - // resolves one, so the overlay half skips any id already claimed. - let manifest_ids: HashSet<&str> = company - .manifest - .agents - .iter() - .map(|a| a.id.as_str()) - .collect(); - let overlay_as_manifest: Vec = overlay_agents - .iter() - .filter(|overlay| !manifest_ids.contains(overlay.id.as_str())) - .map(overlay_agent_to_manifest) - .collect(); - - let mut routed = HashMap::new(); - for agent in company.manifest.agents.iter().chain(&overlay_as_manifest) { - match crate::company::context_routing::resolve_routed_documents( - workspace.as_ref(), - &company.id, - agent, - ) - .await - { - // An agent that resolved nothing is left out of the map rather - // than stored as an empty vec: `build_roster` reads an absent id - // as "no routed documents", so the two are the same answer and - // the map stays the size of what actually routed. - Ok(documents) if documents.is_empty() => {} - Ok(documents) => { - routed.insert(agent.id.clone(), documents); - } - Err(err) => tracing::warn!( - company = %company.id, - agent = %agent.id, - error = %err, - "[context] could not read this role's routed documents; its prompt \ - goes out without them" - ), - } - } - routed - } - - async fn resolve_effective_overlay( - &self, - company: &CompanyRecord, - deps: &HarnessDeps, - ) -> EffectiveOverlay { - match deps.store.load(&company.id).await { - Ok(Some(record)) => EffectiveOverlay { - agents: record.overlay_agents, - budgets: record.overlay_budgets, - policy: record.overlay_policy, - desks: record.overlay_desks, - desk_members: record.overlay_desk_members, - desk_tools: record.overlay_desk_tools, - }, - _ => EffectiveOverlay { - agents: company.overlay_agents.clone(), - budgets: company.overlay_budgets.clone(), - policy: company.overlay_policy.clone(), - desks: company.overlay_desks.clone(), - desk_members: company.overlay_desk_members.clone(), - desk_tools: company.overlay_desk_tools.clone(), - }, - } - } - - /// The current MCP fingerprint for a company (test-only), so a freshness test - /// can assert a rebuild happened without introspecting agent internals. - #[cfg(test)] - pub async fn mcp_fingerprint_of(&self, company: &CompanyId) -> Option { - self.mcp_fingerprints.read().await.get(company).copied() - } - - /// The current overlay-agent fingerprint for a company (test-only), mirroring - /// [`Self::mcp_fingerprint_of`]. - #[cfg(test)] - pub async fn overlay_fingerprint_of(&self, company: &CompanyId) -> Option { - self.overlay_fingerprints.read().await.get(company).copied() - } - - /// The current capability-filter fingerprint for a company (test-only), so a - /// budget-freshness test can assert a rebuild happened (issue #108). - #[cfg(test)] - pub async fn capability_fingerprint_of(&self, company: &CompanyId) -> Option { - self.capability_fingerprints - .read() - .await - .get(company) - .copied() - } - - /// The current bound-repository fingerprint for a company (test-only), so a - /// bind / rotate / revoke freshness test can assert the roster was actually - /// rebuilt rather than inferring it (issue #245). - #[cfg(test)] - pub async fn repo_fingerprint_of(&self, company: &CompanyId) -> Option { - self.repo_fingerprints.read().await.get(company).copied() - } - - /// The current skill-delta fingerprint for a company (test-only), so a - /// skill-freshness test can assert a rebuild happened (issue #41). - #[cfg(test)] - pub async fn skill_fingerprint_of(&self, company: &CompanyId) -> Option { - self.skill_fingerprints.read().await.get(company).copied() - } - - /// The current budget-override fingerprint for a company (test-only), so a - /// budget-freshness test can assert the roster was actually rebuilt after a - /// console cap change rather than inferring it from the refusal (issue - /// #343). This is the observable that makes "no restart" testable. - #[cfg(test)] - pub async fn budget_fingerprint_of(&self, company: &CompanyId) -> Option { - self.budget_fingerprints.read().await.get(company).copied() - } - - /// The current billing-connection fingerprint for a company (test-only), so - /// a credential-freshness test can assert the roster was rebuilt after a key - /// was saved or rotated in Settings → Billing rather than inferring it from - /// the tool list (issues #788, #789). - #[cfg(test)] - pub async fn billing_fingerprint_of(&self, company: &CompanyId) -> Option { - self.billing_fingerprints.read().await.get(company).copied() - } - - /// The current desk-scope fingerprint for a company (test-only), so a - /// desk-scoping test can assert the roster was actually rebuilt after a - /// ceiling or seating change rather than inferring it from a refused call. - #[cfg(test)] - pub async fn desk_fingerprint_of(&self, company: &CompanyId) -> Option { - self.desk_fingerprints.read().await.get(company).copied() - } - - /// The current routed-context fingerprint for a company (test-only), so a - /// routing test can assert that editing a routed workspace note actually - /// rebuilt the roster rather than inferring it from a reply. - #[cfg(test)] - pub async fn context_fingerprint_of(&self, company: &CompanyId) -> Option { - self.context_fingerprints.read().await.get(company).copied() - } - - /// Routes a message to one agent and returns its reply, recording the turn's - /// cost. `agent_id` must name a member of the company's roster. - /// - /// Desk routing (which agent answers a group chat) is the caller's job — v1 - /// is single-responder and the WS3 chat handler picks the addressed member. - /// - /// `chat_id` is the chat/desk **thread** this turn answers (the id journaled - /// as `AgentReply.chat_id`). It rides each live turn-stream frame so the - /// console routes the in-flight tool timeline to the right thread; `None` - /// falls back to the default desk, matching the durable reply. - pub async fn run( - &self, - company: &CompanyId, - agent_id: &str, - message: &str, - deps: &HarnessDeps, - chat_id: Option<&str>, - ) -> crate::Result { - self.run_inner( - company, - agent_id, - message, - deps, - None, - LiveStream::On { chat_id }, - None, - ) - .await - } - - /// Like [`run`](Self::run) but WITHOUT live turn streaming — for a turn that - /// surfaces no operator chat bubble (a workflow agent node, which drops its - /// steps). Its transient `tool_call`/`tool_result` frames would otherwise - /// leak onto the console's live timeline and misattribute to whatever thread - /// most recently sent, so this path publishes nothing (#125 review). - pub async fn run_background( - &self, - company: &CompanyId, - agent_id: &str, - message: &str, - deps: &HarnessDeps, - ) -> crate::Result { - self.run_inner( - company, - agent_id, - message, - deps, - None, - LiveStream::Off, - None, - ) - .await - } - - /// Routes a message to one agent with an operator **steer** control installed - /// (issue #111), so a dispatched task / desk delegation can be paused, - /// cancelled, or redirected mid-flight. Otherwise identical to - /// [`run`](Self::run) — same retrieve→inject, cost accounting, and - /// memory-writeback. The steer hook fires only between tool-loop iterations. - /// `chat_id` routes the live turn-stream frames exactly as in [`run`](Self::run). - /// - /// `run_sink` is the dispatched attempt this turn belongs to, when it - /// belongs to one (issue #242) — a desk turn a *dispatched card* handed its - /// work to records into the card's run, while the same delegation reached - /// from operator chat passes `None`. - #[allow(clippy::too_many_arguments)] - pub async fn run_steered( - &self, - company: &CompanyId, - agent_id: &str, - message: &str, - deps: &HarnessDeps, - control: &SteerControl, - chat_id: Option<&str>, - run_sink: Option>, - ) -> crate::Result { - self.run_inner( - company, - agent_id, - message, - deps, - Some(control), - LiveStream::On { chat_id }, - run_sink, - ) - .await - } - - /// Like [`run_steered`](Self::run_steered) but WITHOUT live turn streaming — - /// for a dispatched task card, which discards its steps and shows no chat - /// bubble. Its transient turn frames must not reach the live console - /// timeline (they'd misattribute to a chat thread), so this path publishes - /// nothing while still honouring the operator steer control (#125 review). - pub async fn run_steered_background( - &self, - company: &CompanyId, - agent_id: &str, - message: &str, - deps: &HarnessDeps, - control: &SteerControl, - run_sink: Option>, - ) -> crate::Result { - self.run_inner( - company, - agent_id, - message, - deps, - Some(control), - LiveStream::Off, - run_sink, - ) - .await - } - - /// The plan-level total-token ceiling, as a refusal or nothing. - /// - /// Extracted from [`run_inner`](Self::run_inner) so the confined turn - /// (issue #416) is gated by the *same* ceiling rather than a second copy of - /// the rule: a turn that reaches nothing still spends model tokens, so a - /// tenant past its cap must not be able to keep spending through the - /// copilot. - async fn total_ceiling_refusal( - company: &CompanyId, - agent_id: &str, - deps: &HarnessDeps, - ) -> Option { - let plan = deps.plan.as_ref()?; - plan.total_budget?; - match deps.meter.as_deref() { - Some(meter) => { - let since = plan.period.period_start_millis(crate::ports::now_millis()); - match meter.query(company, since).await { - Ok(samples) => { - let spent = capability_budget::tokens_in(&samples); - if plan.total_exhausted(spent) { - tracing::info!( - company = %company, - agent = agent_id, - spent, - "[capability-budget] total token ceiling reached; refusing dispatch (no model call) until the period resets" - ); - return Some(TurnOutcome { - reply: TOTAL_BUDGET_EXHAUSTED_NOTICE.to_string(), - steps: Vec::new(), - // No model call ran, so no cap was reached - // (issue #926). A refusal is not a pause. - hit_iteration_cap: false, - // And no in-turn hook fired, because no turn - // ran (issue #1032). The reply already IS the - // budget notice; labelling this as a halt too - // would tell the operator the same thing twice. - halted_for_spend: None, - }); - } - } - Err(error) => { - tracing::warn!( - company = %company, - %error, - "[capability-budget] total-ceiling spend query failed; not hard-refusing — deferring to the per-namespace fail-closed roster" - ); - } - } - } - None => { - tracing::warn!( - company = %company, - "[capability-budget] no usage meter; cannot enforce the total token ceiling — deferring to the per-namespace fail-closed roster" - ); - } - } - None - } - - /// Runs one **confined** turn (issue #416): an ephemeral agent with no - /// tools, no company memory and no roster identity, for a question about one - /// object rather than about the company. - /// - /// Deliberately not a variant of [`run_inner`](Self::run_inner), because the - /// two differ in what they are allowed to touch rather than in a flag: - /// - /// * the agent is **built here and dropped after**, so it is never in the - /// pooled roster and cannot be addressed, dispatched or delegated to; - /// * there is **no retrieve→inject** — the company's prior task outcomes are - /// not prepended to the message, so the model cannot answer from work it - /// was not asked about; - /// * there is **no memory writeback** — the exchange leaves nothing for a - /// later company turn to retrieve, so a confined conversation cannot - /// become unconfined context tomorrow. - /// - /// What it does share: the plan-level token ceiling (spend is spend), live - /// turn streaming onto the addressed thread, and cost recording, so a - /// confined turn is billed and observable exactly like any other. - pub async fn run_confined( - &self, - company: &CompanyId, - company_name: &str, - message: &str, - deps: &HarnessDeps, - chat_id: Option<&str>, - confinement: &confine::Confinement, - ) -> crate::Result { - if let Some(refusal) = - Self::total_ceiling_refusal(company, confine::CONFINED_AGENT_ID, deps).await - { - return Ok(refusal); - } - - let agent = CompanyAgent { - agent_id: confine::CONFINED_AGENT_ID.to_string(), - role: "Workflow copilot".to_string(), - // A confined turn carries no manifest teammate, so there is no - // per-agent daily cap to read; the company-wide ceiling above is the - // one that applies to it. - budget_usd_daily: None, - agent: Mutex::new(confine::build_confined_agent( - company, - company_name, - confinement, - deps, - )?), - }; - - let stream_ctx = Some(crate::turn_stream::TurnStreamCtx { - company: company.clone(), - agent_id: confine::CONFINED_AGENT_ID.to_string(), - chat_id: chat_id - .map(str::to_string) - .unwrap_or_else(|| crate::server::ops::language::DEFAULT_DESK.to_string()), - }); - - // The message goes to the model AS SENT. This is the retrieve→inject - // step's absence, and it is the difference between "grounded in one - // workflow" and "confined to one workflow". - let (outcome, turn_costs) = agent - .run_with_steer(message, None, stream_ctx, None) - .await?; - - let provider_slug = deps.provider.telemetry_provider_id(); - for turn_cost in &turn_costs { - record_turn_cost( - turn_cost, - confine::CONFINED_AGENT_ID, - &provider_slug, - company, - deps.store.as_ref(), - deps.meter.as_deref(), - None, - ) - .await?; - } - - Ok(outcome) - } - - #[allow(clippy::too_many_arguments)] - async fn run_inner( - &self, - company: &CompanyId, - agent_id: &str, - message: &str, - deps: &HarnessDeps, - steer: Option<&SteerControl>, - live: LiveStream<'_>, - run_sink: Option>, - ) -> crate::Result { - let agent = { - let guard = self.agents.read().await; - let roster = guard - .get(company) - .ok_or_else(|| OpenCompanyError::CompanyNotFound(company.to_string()))?; - roster - .iter() - .find(|a| a.agent_id == agent_id) - .cloned() - .ok_or_else(|| { - OpenCompanyError::InvalidRequest(format!( - "agent '{agent_id}' is not on company '{company}' roster" - )) - })? - }; - - // Renew the agent's sandbox directory at the moment it acts (issue - // #409). `build_agent` already created it, but a roster is built once - // and then cached behind fingerprints — and handed *across* an in-place - // rebuild — so a workspace that goes missing afterwards (a restored or - // wiped data dir, an operator clearing the tree, a boot that raced a - // not-yet-mounted volume) would otherwise stay missing for the life of - // the process, and every relative file write would be refused as if it - // had tried to escape the sandbox. Two syscalls on the already-exists - // path, against a turn that is about to call a model — not worth - // deferring off the runtime thread. - // - // Deliberately not fatal, for the same reason `build_agent`'s attempt is - // not: an agent with no file grant runs a perfectly good turn without - // this directory. The `error!` (not `warn!`) records the one condition - // under which the misdirecting guard message can still be reached, so it - // is greppable next to the refusal it explains. Both of those are the - // right calls and issue #449 does not change either. - // - // What #449 changes is only how often it is *said*. A workspace root - // that cannot be written — a volume that failed to mount, a path that - // resolves onto a file — fails identically on every dispatch, so the - // unconditional `error!` emitted one byte-identical line per turn, - // forever, with nothing distinguishing the thousandth from the first. - // The state is edge-triggered instead: the first failure reads exactly - // as it did before, the repeats are silent, and a recovery gets one - // `info!` so a reader who saw the error learns when it ended. The - // attempt itself still runs every dispatch — see - // `note_workspace_attempt` for why memoising it would be a regression. - let attempt = build::ensure_agent_workspace(&deps.workspace_root, company, agent_id); - let report = self.note_workspace_attempt(company, agent_id, attempt.is_err()); - if !report.is_silent() { - let workspace = build::agent_workspace(&deps.workspace_root, company, agent_id); - match attempt { - Err(error) => tracing::error!( - company = %company, - agent = agent_id, - workspace = %workspace.display(), - %error, - "[harness] could not create the agent workspace before dispatch; relative file writes will be refused (the refusal will read as a workspace escape, but the cause is this missing directory)" - ), - Ok(_) => tracing::info!( - company = %company, - agent = agent_id, - workspace = %workspace.display(), - "[harness] agent workspace is available again; the earlier creation failure has cleared and relative file writes work" - ), - } - } - - // Plan-level total-token ceiling (issue #188): a HARD dispatch refusal - // that never reaches the model once the tenant's total period spend - // crosses the cap. The per-namespace budget gate in `ensure` is *soft* — - // it only trims which exec tools the roster carries; an exhausted - // tenant's turn still runs on intrinsic tools and burns model tokens. - // This closes that gap by refusing dispatch outright, before any model - // call, on every path that funnels through `run_inner` (operator chat, - // task, steered/background). We return early here — before retrieve→ - // inject and the memory writeback — so a refused turn costs nothing and - // leaves no fabricated outcome in the memory store. - // - // Fail-closed tradeoff (issue #188): the hard refusal fires ONLY when - // spend is actually readable. With no meter, or a meter whose query - // errors, we do NOT brick the tenant on a transient read failure — we - // fall through to run the turn, which the per-namespace fail-closed path - // in `resolve_filter`/`ensure` has already stripped of every exec tool. - // A `warn!` records the deferral. Refusing every turn on a flaky meter - // read would be a strictly worse failure mode than letting an - // intrinsic-tools-only turn through. - if let Some(refusal) = Self::total_ceiling_refusal(company, agent_id, deps).await { - return Ok(refusal); - } - - // Per-agent daily spend cap (issue #304): the same HARD, pre-model-call - // refusal as the ceiling above, scoped to ONE teammate. - // - // This is the layer that matters most in practice. The manifest's - // `budget_usd_daily` was validated, persisted and passed to - // `ApprovalPolicy` — where it sat on a field with no reader. But the - // dominant spend stream is not tool calls at all, it is inference, and - // inference never reaches a `ToolPolicy`. Gating only priced tool calls - // (the policy arm) would leave a capped teammate free to burn its budget - // many times over on model turns alone, which is how the cap came to be - // decorative in the first place. - // - // Refused BEFORE retrieve→inject and the memory writeback, exactly like - // the total ceiling, so a refused turn costs nothing and leaves no - // fabricated outcome in the store. The reply names the teammate, the cap - // and the reset — never a bare failure. - // - // FAIL-OPEN, mirroring #188's documented tradeoff: with no meter, or a - // meter whose query errors, we warn and run the turn. Bricking a - // company's cognition on a flaky read would be a strictly worse failure - // mode than one day of overspend, and there is no operator recourse at - // turn level (unlike the policy arm, whose park a human can approve — - // which is why THAT layer fails closed and this one does not). - if let Some(cap) = agent.budget_usd_daily { - match deps.meter.as_deref() { - Some(meter) => { - let since = crate::metering::utc_day_start_millis(crate::ports::now_millis()); - match meter.query(company, since).await { - Ok(samples) => { - let spent = crate::metering::usd_spent_by_agent(&samples, agent_id); - if spent >= cap { - tracing::info!( - company = %company, - agent = agent_id, - spent, - cap, - "[agent-budget] daily spend cap reached; refusing dispatch (no model call) until 00:00 UTC" - ); - return Ok(TurnOutcome { - reply: agent_budget_exhausted_notice(agent_id, cap), - steps: Vec::new(), - // No model call ran, so no cap was reached - // (issue #926). A refusal is not a pause. - hit_iteration_cap: false, - // Same teammate cap, refused BEFORE the - // turn (issue #1032). The in-turn brake - // never armed, and the reply above already - // names the cap it refused against. - halted_for_spend: None, - }); - } - } - Err(error) => { - tracing::warn!( - company = %company, - agent = agent_id, - %error, - "[agent-budget] daily-spend query failed; running the turn rather than bricking this teammate" - ); - } - } - } - None => { - tracing::warn!( - company = %company, - agent = agent_id, - "[agent-budget] no usage meter; the per-agent daily spend cap cannot be enforced on this host" - ); - } - } - } - - // Retrieve→inject: pull the top-K prior task outcomes relevant to this - // message and prepend them as context. On a cold store this yields no - // hits and the message is passed through unchanged. - let hits = deps - .context - .search(company, message, memory_loop::RETRIEVE_TOP_K) - .await?; - let augmented = memory_loop::inject(message, &hits); - - // Run the turn and record its real cost. `CompanyAgent::run` reads each - // attempt's token/cost totals from openhuman's public `last_turn_usage()` - // accessor and returns one entry per attempt (two when the empty-response - // wrapper retried once). A zero-usage attempt (offline provider) writes - // nothing, so the inert-metering contract holds. - // Live tool-call streaming: for a turn that surfaces an operator chat - // bubble (`live`), hand the runner the routing context so it tees each - // progress event onto the company's transient turn-stream bus as it - // happens (the console renders the timeline live). Background turns — - // dispatched task cards and workflow agent nodes — pass `live = false` - // and stream nothing, since they carry no chat thread to render onto and - // their frames would otherwise misattribute to the active chat (#125 - // review). Either way the durable `TurnStep`s still fold from the same - // buffered events at turn end. - let stream_ctx = match live { - LiveStream::On { chat_id } => Some(crate::turn_stream::TurnStreamCtx { - company: company.clone(), - agent_id: agent_id.to_string(), - // The chat/desk thread this turn answers — the same id journaled - // as `AgentReply.chat_id`, so the console keys the live timeline - // on it and concurrent turns on different threads never - // cross-attribute. Falls back to the default desk to match the - // durable reply when the caller addressed no desk (e.g. an API - // client that omits `chat`). - chat_id: chat_id - .map(str::to_string) - .unwrap_or_else(|| crate::server::ops::language::DEFAULT_DESK.to_string()), - }), - LiveStream::Off => None, - }; - let (outcome, turn_costs) = agent - .run_with_steer(&augmented, steer, stream_ctx, run_sink.clone()) - .await?; - // Issue #242: fold this turn's spend into the attempt it belongs to. - // Per turn, not once at the end, so a redirect re-run and a delegate's - // turn both count — an attempt's cost is what the attempt spent. This is - // a second *reader* of `turn_costs`, not a second writer: the ledger and - // the usage meter below stay the only places money is recorded. - if let Some(sink) = run_sink.as_ref() { - for turn_cost in &turn_costs { - sink.add_usage(turn_cost); - } - } - // Attribute cost to the provider this turn actually resolved to. With a - // per-tenant [`TenantProvider`](crate::harness::provider::TenantProvider) - // a console BYOK switch changes the slug between turns, so read it live - // rather than trusting the static `deps.provider_slug` baked at build. - let provider_slug = deps.provider.telemetry_provider_id(); - for turn_cost in &turn_costs { - record_turn_cost( - turn_cost, - agent_id, - &provider_slug, - company, - deps.store.as_ref(), - deps.meter.as_deref(), - // Issue #242: attribute the sample to the attempt this turn ran - // under, so "what did this run cost?" is answerable from the - // meter as well as from the run row. - run_sink.as_ref().map(|s| s.run_id()), - ) - .await?; - } - - // Store: persist the outcome (original task + reply) so it compounds - // into later turns. Without this the harness never writes memory back. - // SECURITY: the reply **text only** — the scrubbed `outcome.steps` never - // enter the memory store, so a step detail can never be retrieved and - // re-injected into a later turn. - if !matches!( - steer.and_then(SteerControl::pending), - Some(SteerAction::Cancel) - ) { - deps.context - .put( - company, - memory_loop::outcome_chunk(agent_id, message, &outcome.reply), - ) - .await?; - } - - Ok(outcome) - } - - /// Number of companies currently resident in the pool (test/observability). - pub async fn resident_companies(&self) -> usize { - self.agents.read().await.len() - } -} - -/// A stable fingerprint of an effective MCP server set, used to detect a console -/// change (add / remove / enable-toggle / token rotation) between -/// [`HarnessPool::ensure`] calls. Hashes only non-secret configuration plus the -/// credential substrings — the resulting `u64` is non-reversible and never -/// surfaces anywhere, so it is not a credential leak, and hashing the credential -/// substrings means a rotate-token also invalidates the cached roster. -fn mcp_fingerprint(decls: &[McpServerDecl]) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let mut hasher = DefaultHasher::new(); - decls.len().hash(&mut hasher); - for decl in decls { - decl.name.hash(&mut hasher); - decl.endpoint.hash(&mut hasher); - decl.enabled.hash(&mut hasher); - decl.description.hash(&mut hasher); - decl.allowed_tools.hash(&mut hasher); - decl.disallowed_tools.hash(&mut hasher); - decl.timeout_secs.hash(&mut hasher); - auth_kind(&decl.auth).hash(&mut hasher); - for secret in decl.auth.secret_values() { - secret.hash(&mut hasher); - } - } - hasher.finish() -} - -/// A small discriminant for an [`AuthMaterial`] variant, for the fingerprint. -fn auth_kind(material: &crate::company::mcp::AuthMaterial) -> u8 { - use crate::company::mcp::AuthMaterial::*; - match material { - None => 0, - Bearer(_) => 1, - Header { .. } => 2, - QueryParam { .. } => 3, - OAuth { .. } => 4, - } -} - -/// Refreshes any near-expiry console-OAuth credential in `decls` before the -/// registry is built, re-persisting the rotated token **write-only** so agents -/// never send an expired bearer. Per-tenant analogue of OpenHuman's -/// `mcp_registry::oauth::refresh_if_expired`. A refresh failure is non-fatal — -/// the old token is kept and the next `401` re-prompts sign-in. -#[cfg(feature = "mcp")] -async fn refresh_oauth_decls( - company: &CompanyId, - decls: &mut [McpServerDecl], - secrets: &dyn SecretStore, -) { - use crate::company::mcp_oauth; - - for decl in decls.iter_mut() { - if !mcp_oauth::needs_refresh(&decl.auth, 60) { - continue; - } - let Some(new_material) = mcp_oauth::refresh(&decl.auth).await else { - continue; - }; - match crate::company::mcp::store_auth(company, &decl.name, &new_material, secrets).await { - Ok(()) => decl.auth = new_material, - Err(err) => log::warn!( - "[mcp-oauth] failed to persist refreshed token for `{}`: {}", - decl.name, - err.code() - ), - } - } -} - -/// Without the `mcp` feature there is no OAuth credential to refresh, so this is -/// a no-op (keeps `resolve_effective_mcp` uniform across the two builds). -#[cfg(not(feature = "mcp"))] -async fn refresh_oauth_decls( - _company: &CompanyId, - _decls: &mut [McpServerDecl], - _secrets: &dyn SecretStore, -) { -} - -/// A stable fingerprint of an overlay-agent set (issue #71), used to detect a -/// teammate add/remove/edit between [`HarnessPool::ensure`] calls. Mirrors -/// [`mcp_fingerprint`]'s shape; no secrets are involved here so there is -/// nothing to scrub — an [`OverlayAgent`] is display data. -fn overlay_fingerprint(agents: &[OverlayAgent]) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - agents.len().hash(&mut hasher); - for agent in agents { - agent.id.hash(&mut hasher); - agent.name.hash(&mut hasher); - agent.role.hash(&mut hasher); - agent.description.hash(&mut hasher); - // Issue #661 / L5: a grant edit changes the roster the harness must - // build, so it has to move this fingerprint — otherwise a re-grant would - // persist and be silently ignored until the next process restart, the - // same staleness the tier/skill fingerprints exist to prevent. Hashed in - // order (an operator's own list), length folded in first via the slice - // length above so `["a","b"]` cannot collide with `["ab"]`. - agent.tools.hash(&mut hasher); - } - hasher.finish() -} - -/// A stable hash of the operator's `[policy]` override, so a console tier change -/// rebuilds the roster on the company's next `ensure` (issue #562). -/// -/// # Why this axis has to exist at all -/// -/// `ApprovalPolicy` is constructed in [`build_roster`], **once per roster -/// build** — not once per call. The roster is cached and rebuilt only when one -/// of the fingerprints in the staleness check moves. So without this function a -/// console tier change would be written, persisted, and then **silently ignored -/// until the process restarted**: the write route would return `204`, the -/// console would show the new tier, and every agent would keep running the old -/// one. That is the same failure the skill-delta fingerprint above exists to -/// prevent, and it is invisible from the outside. -/// -/// # What is hashed, and what deliberately is not -/// -/// - `mode` and `always_approve` are hashed — they are what the gate reads. -/// - `always_approve` is hashed **in order**, unlike the budget set. The order -/// is the operator's own list as they wrote it, not an accumulation of -/// independent rows, so a reorder is a real edit rather than a spurious -/// difference. Its length is folded in first so `["a","b"]` cannot collide -/// with `["ab"]`. -/// - The `Some`/`None` distinction is hashed for both fields, because "not -/// overridden" and "overridden to the manifest's current value" must stay -/// apart: the manifest can change under a rebuild, and collapsing them would -/// pin the override to a value the operator never chose. -/// - **Attribution (`set_by`, `at_millis`) is deliberately NOT hashed**, for the -/// same reason the budget fingerprint omits it: who set the tier and when -/// changes nothing an agent can act on, and folding it in would rebuild the -/// roster — dropping live agent sessions — on a save that re-set the same tier. -fn policy_fingerprint(override_: Option<&PolicyOverride>) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - match override_ { - None => 0u8.hash(&mut hasher), - Some(entry) => { - 1u8.hash(&mut hasher); - match &entry.mode { - Some(mode) => { - 1u8.hash(&mut hasher); - mode.hash(&mut hasher); - } - None => 0u8.hash(&mut hasher), - } - match &entry.always_approve { - Some(kinds) => { - 1u8.hash(&mut hasher); - kinds.len().hash(&mut hasher); - for kind in kinds { - kind.hash(&mut hasher); - } - } - None => 0u8.hash(&mut hasher), - } - } - } - hasher.finish() -} - -/// The live overlay state one roster rebuild is resolved against. -/// -/// A struct rather than the tuple this used to be: it grew past the point where -/// positional returns stay readable, and — more to the point — the desk fields -/// were added because desks now decide *capability*, so a caller silently -/// binding `desk_tools` to the `desks` position would hand every teammate the -/// wrong tool belt with nothing to catch it. -pub(crate) struct EffectiveOverlay { - pub agents: Vec, - pub budgets: Vec, - pub policy: Option, - pub desks: Vec, - pub desk_members: Vec, - pub desk_tools: std::collections::BTreeMap>, -} - -/// Fingerprints the routed workspace documents a roster's personas are built -/// from — **over their bodies**, not their names. -/// -/// Hashing the content is the whole point. The routing table is manifest data -/// and does not move when an operator edits a note, so a name-only hash would -/// leave the edit invisible: the persona is assembled once per roster, and the -/// fast path would keep serving a prompt quoting the old text until the process -/// restarted. That is precisely the staleness the routing layer exists to avoid. -/// -/// Sorted by agent id before hashing, for the reason [`budget_fingerprint`] -/// documents — a `HashMap` has no order, and an order-sensitive hash would drop -/// every live agent session on a rebuild that changed nothing. -fn routed_context_fingerprint(routed: &HashMap>) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut ordered: Vec<(&String, &Vec<(String, String)>)> = routed.iter().collect(); - ordered.sort_by(|a, b| a.0.cmp(b.0)); - - let mut hasher = DefaultHasher::new(); - ordered.len().hash(&mut hasher); - for (agent_id, documents) in ordered { - agent_id.hash(&mut hasher); - documents.len().hash(&mut hasher); - for (path, body) in documents { - path.hash(&mut hasher); - body.hash(&mut hasher); - } - } - hasher.finish() -} - -/// Fingerprints the desk scoping a roster's grants are resolved through: which -/// desks exist, who sits on them, and what each one's tool ceiling is. -/// -/// All three axes are hashed together because all three feed one answer — an -/// agent's effective grant. Seating a teammate on a restricted desk narrows its -/// belt just as surely as editing that desk's ceiling does, so a fingerprint -/// over the ceilings alone would leave a membership change invisible until the -/// next restart, which is the staleness bug this whole fingerprint set exists to -/// prevent. -/// -/// Sorted before hashing, for the reason [`budget_fingerprint`] documents: the -/// write routes push and retain rather than maintain an order, and an -/// order-sensitive hash would drop every live agent session on a save that -/// changed nothing an agent can observe. (`desk_tools` is a `BTreeMap` and so is -/// already ordered by construction.) -fn desk_scope_fingerprint( - desks: &[OverlayDesk], - members: &[OverlayDeskMember], - tools: &std::collections::BTreeMap>, -) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - - let mut desk_ids: Vec<&str> = desks.iter().map(|desk| desk.id.as_str()).collect(); - desk_ids.sort_unstable(); - desk_ids.hash(&mut hasher); - - let mut seats: Vec<(&str, &str)> = members - .iter() - .map(|seat| (seat.desk_id.as_str(), seat.agent_id.as_str())) - .collect(); - seats.sort_unstable(); - seats.hash(&mut hasher); - - tools.len().hash(&mut hasher); - for (desk_id, ceiling) in tools { - desk_id.hash(&mut hasher); - ceiling.hash(&mut hasher); - } - - hasher.finish() -} - -/// A stable fingerprint of a company's operator budget-override set (issue -/// #343), used to detect a cap set / changed / cleared / reset between -/// [`HarnessPool::ensure`] calls. Mirrors [`overlay_fingerprint`]'s shape; a -/// [`BudgetOverride`] holds no secret. +/// The ACP `RunTurn`, under the path it had before the split. /// -/// Two details carry weight: -/// -/// - The set is **sorted by `agent_id`** first, because the write routes push -/// and retain rather than maintain an order, and an order-sensitive hash would -/// rebuild the roster (dropping live agent sessions) on a save that changed -/// nothing an agent can observe. -/// - The cap is hashed as an `Option` **discriminant plus `f64::to_bits`**, not -/// through `PartialEq`. `f64` is not `Hash`, and going through bits is also -/// what keeps `Some(0.0)` distinct from `None` in the hash — the very -/// distinction the issue insists must not collapse. `to_bits` additionally -/// makes the hash total over values `PartialEq` would call incomparable. -fn budget_fingerprint(overrides: &[BudgetOverride]) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut ordered: Vec<&BudgetOverride> = overrides.iter().collect(); - ordered.sort_by(|a, b| a.agent_id.cmp(&b.agent_id)); - - let mut hasher = DefaultHasher::new(); - ordered.len().hash(&mut hasher); - for entry in ordered { - entry.agent_id.hash(&mut hasher); - match entry.budget_usd_daily { - Some(cap) => { - 1u8.hash(&mut hasher); - cap.to_bits().hash(&mut hasher); - } - None => 0u8.hash(&mut hasher), - } - } - // Attribution is deliberately NOT hashed: who set the cap and when changes - // nothing an agent can act on, and folding it in would rebuild the roster - // (discarding live sessions) every time the same value was re-saved. - hasher.finish() -} - -/// A stable fingerprint of a company's operator skill-delta set (issue #41), -/// used to detect a skill authored / edited / enabled / disabled between -/// [`HarnessPool::ensure`] calls. Mirrors [`mcp_fingerprint`]'s shape. -/// -/// The deltas are **sorted by `slug`** before hashing because -/// [`SkillStateStore::list`](crate::ports::skills_state::SkillStateStore::list) -/// gives no ordering contract — an order-sensitive hash would thrash the roster -/// (and drop live agent conversation state) whenever the store returned the -/// same skills in a different row order. The full `custom_doc` body is hashed so -/// an *edited* skill (same slug, new content) also triggers a rebuild. No -/// The disabling [`SkillState`] deltas a company's `[globals].disable` implies. -/// -/// One per `skill:` entry, and nothing else: an entry naming another kind -/// is that kind's business, and manifest validation has already refused an entry -/// naming nothing at all. -pub(crate) fn globals_skill_disables(disable: &[String]) -> Vec { - disable - .iter() - .filter_map(|entry| entry.strip_prefix("skill:")) - .map(|slug| SkillState { - slug: slug.to_string(), - enabled: false, - // The shared library is where these skills are authored, so that is - // what they are a delta over. The value is inert here in any case: - // this delta is synthesized per rebuild, never stored, and only its - // `enabled = false` is read. - source: crate::ports::SkillSource::Registry, - custom_doc: None, - }) - .collect() -} - -/// secrets are involved — a skill delta is operator-authored content. -fn skill_delta_fingerprint(deltas: &[SkillState]) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut ordered: Vec<&SkillState> = deltas.iter().collect(); - ordered.sort_by(|a, b| a.slug.cmp(&b.slug)); - - let mut hasher = DefaultHasher::new(); - ordered.len().hash(&mut hasher); - for delta in ordered { - delta.slug.hash(&mut hasher); - delta.enabled.hash(&mut hasher); - delta.source.hash(&mut hasher); - delta.custom_doc.hash(&mut hasher); - } - hasher.finish() -} - -/// Fingerprint of a company's bound repositories (issue #245). -/// -/// Over `(key, token_fingerprint, branches)`, sorted by key, because those are -/// exactly the three things a rebuild has to notice: -/// -/// * **key** — a bind adds one, a revoke removes one, and either changes what -/// `repo_checkout` can resolve and what its description names; -/// * **token fingerprint** — a rotation leaves the key alone, and a *revoked* -/// credential blanks it while the key survives, so keying on the set of -/// repositories would leave an agent holding a tool over a binding that can no -/// longer fetch; -/// * **branches** — the set a checkout may name, and the only other field the -/// tools read. -/// -/// Deliberately not `size_bytes` or `last_fetched_millis`: both move on every -/// fetch, and a fetch is something the agent's own tool does — folding them in -/// would rebuild the roster after every checkout, for no change an agent can -/// observe. -fn repo_binding_fingerprint(bindings: &[crate::runtime::repo_manager::types::RepoBinding]) -> u64 { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - - let mut ordered: Vec<&crate::runtime::repo_manager::types::RepoBinding> = - bindings.iter().collect(); - ordered.sort_by(|a, b| a.key.cmp(&b.key)); - - let mut hasher = DefaultHasher::new(); - ordered.len().hash(&mut hasher); - for binding in ordered { - binding.key.hash(&mut hasher); - binding.token_fingerprint.hash(&mut hasher); - binding.branches.hash(&mut hasher); - } - hasher.finish() -} - -/// Build every roster agent for a company: every manifest `[[agent]]`, plus -/// every operator- or orchestrator-added [`OverlayAgent`] (issue #71 — Active -/// Runtime Teammates) that does not collide with a manifest agent id. -/// -/// Overlay teammates were presentation-only before this cell (listed in the -/// console Team tab but never addressable); this promotes each one into a real -/// [`CompanyAgent`] with the same shape [`build::build_agent`] gives a manifest -/// agent — a standard (company-wide) tool grant, no cognition tier (the -/// default `chat-v1` model), and never the orchestrator. A manifest agent -/// always wins an id collision: the version-controlled roster is authoritative, -/// and [`orchestrator::orchestrator_id`] only ever looks at `manifest.agents`, -/// so an overlay teammate can never become the orchestrator. -/// -/// `skill_deltas` are the company's operator skill overrides (fetched once by -/// the async caller); every agent folds them into its effective skill set. -/// -/// `routed_context` maps an agent id to the workspace documents routed into its -/// system prompt, resolved by the async caller for the same reason -/// `skill_deltas` is — this function is synchronous and the `WorkspaceStore` is -/// not. An agent absent from the map gets no routed documents, which is the -/// correct reading for a company with no workspace store wired: fail closed to -/// the pre-routing prompt rather than to a half-populated one. -pub(crate) fn build_roster( - company: &CompanyRecord, - deps: &HarnessDeps, - skill_deltas: &[SkillState], - routed_context: &HashMap>, -) -> crate::Result>> { - // Issue #562: the policy in force, not the one the manifest shipped with — - // the same relationship `effective_budget` (issue #343) has to the manifest - // cap, and resolved through the same record so the console and this gate - // cannot disagree about which tier is live. - // - // Owned rather than borrowed because the effective value is a field-wise - // merge of the override and the manifest, so there may be nothing to borrow. - let effective = company.effective_policy(); - let policy: &Policy = &effective; - let company_name = &company.manifest.company.name; - let allow = &company.manifest.tools.allow; - // The orchestrator agent (tier `orchestrator`, else the first agent) receives - // the delegating-orchestrator persona + tools (issue #53). - let orchestrator = orchestrator::orchestrator_id(&company.manifest.agents); - - let mut roster = - Vec::with_capacity(company.manifest.agents.len() + company.overlay_agents.len()); - - for manifest_agent in &company.manifest.agents { - // Issue #343: the cap in force, not the one the manifest shipped with. - // `effective_budget` is an operator override when one is stored and the - // manifest value otherwise, so a console cap change reaches BOTH readers - // built below — the `ApprovalPolicy` arm and `CompanyAgent`'s copy that - // the L1 dispatch gate reads — from this one call. - let effective_budget = company.effective_budget(&manifest_agent.id); - let mut agent_policy = ApprovalPolicy::new(policy, effective_budget) - .with_requests(deps.approval_requests.clone()) - // Issue #243: stamp who the parked effect belongs to, so approving it - // can hand the grant back to this agent rather than to nobody. - .with_agent(manifest_agent.id.clone()); - // Issue #304: give the policy something to measure `budget_usd_daily` - // against. Only wired when the host has a meter — without one the cap - // arm stays inert and warns once, rather than parking every priced call - // on a host that can never answer the question. - if let Some(meter) = deps.meter.as_ref() { - agent_policy = agent_policy.with_spend(meter.clone(), company.id.clone()); - } - let is_orchestrator = orchestrator.as_deref() == Some(manifest_agent.id.as_str()); - // Three-level narrowing: company → the desks this teammate sits on → - // the teammate itself. `agent_desk_tools` resolves through the record's - // *effective* desk membership, so a console-seated member is scoped by - // its desk exactly as a manifest one is. - let desk_tools = company.agent_desk_tools(&manifest_agent.id); - let desk_allows: Vec<&[String]> = desk_tools.iter().map(Vec::as_slice).collect(); - let grants = agent_scoped_grants(allow, &desk_allows, &manifest_agent.tools); - let agent = build::build_agent( - &company.id, - company_name, - manifest_agent, - agent_policy, - deps, - &grants, - skill_deltas, - routed_context - .get(&manifest_agent.id) - .map(Vec::as_slice) - .unwrap_or(&[]), - is_orchestrator, - )?; - roster.push(Arc::new(CompanyAgent { - agent_id: manifest_agent.id.clone(), - role: manifest_agent.role.clone(), - budget_usd_daily: effective_budget, - agent: Mutex::new(agent), - })); - } - - // Issue #71 — Active Runtime Teammates (minimal slice): promote every - // operator/orchestrator-added overlay teammate into a real roster agent - // too, skipping any id already claimed by a manifest agent. - let manifest_ids: HashSet<&str> = company - .manifest - .agents - .iter() - .map(|a| a.id.as_str()) - .collect(); - for overlay in &company.overlay_agents { - if manifest_ids.contains(overlay.id.as_str()) { - continue; - } - let manifest_agent = overlay_agent_to_manifest(overlay); - // Issue #343: an overlay teammate has no manifest row to carry a cap, so - // before the override existed it was unconditionally uncapped — the "v1 - // limitation" this lifts. `effective_budget` gives it a stored cap when - // an operator set one, and `None` (as before) when nobody has. - let effective_budget = company.effective_budget(&manifest_agent.id); - let mut agent_policy = ApprovalPolicy::new(policy, effective_budget) - .with_requests(deps.approval_requests.clone()) - // An overlay teammate is a real roster agent and re-dispatches the - // same way a manifest one does (issue #243). - .with_agent(manifest_agent.id.clone()); - if let Some(meter) = deps.meter.as_ref() { - agent_policy = agent_policy.with_spend(meter.clone(), company.id.clone()); - } - // An overlay teammate is scoped by its desks the same as a manifest one: - // it can be seated on a desk, and a desk ceiling that applied to only - // half its members would not be a ceiling. - let desk_tools = company.agent_desk_tools(&manifest_agent.id); - let desk_allows: Vec<&[String]> = desk_tools.iter().map(Vec::as_slice).collect(); - let grants = agent_scoped_grants(allow, &desk_allows, &manifest_agent.tools); - let agent = build::build_agent( - &company.id, - company_name, - &manifest_agent, - agent_policy, - deps, - &grants, - skill_deltas, - routed_context - .get(&manifest_agent.id) - .map(Vec::as_slice) - .unwrap_or(&[]), - /* is_orchestrator */ false, - )?; - roster.push(Arc::new(CompanyAgent { - agent_id: manifest_agent.id.clone(), - role: manifest_agent.role.clone(), - budget_usd_daily: effective_budget, - agent: Mutex::new(agent), - })); - } - - Ok(roster) -} - -/// Converts an operator-added [`OverlayAgent`] into the manifest agent shape -/// [`build::build_agent`] consumes: an empty `tools` list (so -/// [`agent_effective_grants`] falls back to the full company `[tools].allow` -/// — the "standard tool grant"), no cognition tier (→ the default `chat-v1` -/// model), and no manifest budget cap — an overlay teammate has no manifest row -/// at all, so its cap (if any) comes from the record's budget overrides via -/// [`CompanyRecord::effective_budget`], resolved by the caller. The overlay's -/// `name` is carried across (issue #1105): it is what -/// [`crate::metering::roster_display_names`] labels this teammate with -/// everywhere in the console, so -/// [`persona_prompt`](crate::company::prompt::persona_prompt) needs it to frame the -/// agent as the person the operator is addressing. Dropping it here — as this -/// did until #1105 — left the model knowing only its role, so it denied being -/// the name on its own DM header. -fn overlay_agent_to_manifest(overlay: &OverlayAgent) -> ManifestAgent { - ManifestAgent { - global: false, - id: overlay.id.clone(), - role: overlay.role.clone(), - name: Some(overlay.name.clone()), - description: overlay.description.clone(), - tier: None, - // Issue #661 / L5: carry the overlay's own per-teammate grant. An empty - // list here is unchanged behaviour — `agent_effective_grants` reads it as - // the standard company-wide grant, exactly as the hardcoded empty did. - // A non-empty list is intersected with `[tools].allow` by that same - // function below (narrow-only, never a widen). - tools: overlay.tools.clone(), - // Issue #176: an overlay teammate declares no delegation allowlist in - // this slice, so it carries today's behaviour — no hand-off tools wired. - // Opting overlays in needs a console write surface; see the follow-up. - delegates_to: Vec::new(), - context: None, - budget_usd_daily: None, - prompt: None, - prompt_files: Vec::new(), - prompt_files_resolved: Vec::new(), - classes: Vec::new(), - ledgers: None, - can_declare_ledgers: true, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Mutex as StdMutex; - - use async_trait::async_trait; - use tinyagents::harness::model::{ChatModel, ModelRequest, ModelResponse}; - - use crate::company::CompanyManifest; - use crate::harness::provider::MockProvider; - use crate::ports::UsageSample; - use crate::ports::types::{ - ChunkAddr, ChunkHit, ChunkMeta, CompanySummary, ContextChunk, LedgerEntry, - }; - // The two-level resolver. Test-only now: the roster build goes through - // `agent_scoped_grants`, and these tests assert the desk-less case still - // resolves identically to what shipped before desks could scope tools. - use crate::runtime::builder::agent_effective_grants; - - fn fp_entry(mode: Option<&str>, always: Option>) -> PolicyOverride { - use crate::ports::types::{Actor, ActorKind}; - PolicyOverride { - mode: mode.map(str::to_string), - always_approve: always.map(|v| v.into_iter().map(str::to_string).collect()), - set_by: Actor { - kind: ActorKind::User, - id: "user-1".to_string(), - }, - at_millis: 1_700_000_000_000, - } - } - - /// The fingerprint moves when the tier moves (issue #562). - /// - /// This is the assertion that keeps the feature from being a no-op. - /// `ApprovalPolicy` is built once per roster build, and `ensure` reuses the - /// cached roster unless a fingerprint changed — so if this returned a - /// constant, a console tier change would persist, return `204`, render as - /// applied, and be **silently ignored until the process restarted**. Every - /// other test in this change would still pass. - #[test] - fn the_policy_fingerprint_moves_when_the_tier_does() { - let none = policy_fingerprint(None); - let supervised = policy_fingerprint(Some(&fp_entry(Some("supervised"), None))); - let full = policy_fingerprint(Some(&fp_entry(Some("full"), None))); - - assert_ne!( - supervised, full, - "a tier change must move the fingerprint or the roster is never rebuilt" - ); - assert_ne!( - none, supervised, - "setting an override must move the fingerprint even when it names the \ - tier the manifest already had — the manifest can change under a rebuild" - ); - } - - /// An always-ask edit moves it too, including clearing the list. - /// - /// `always_approve` wins over every tier including `full`, so an edit that - /// did not rebuild would leave the gate enforcing a list the operator had - /// already changed — the failure mode is stricter *or* looser than what the - /// console shows, depending on the edit. - #[test] - fn the_policy_fingerprint_moves_when_the_always_ask_list_does() { - let absent = policy_fingerprint(Some(&fp_entry(Some("auto"), None))); - let empty = policy_fingerprint(Some(&fp_entry(Some("auto"), Some(vec![])))); - let one = policy_fingerprint(Some(&fp_entry(Some("auto"), Some(vec!["payment.send"])))); - let two = policy_fingerprint(Some(&fp_entry( - Some("auto"), - Some(vec!["payment.send", "filing.submit"]), - ))); - - assert_ne!( - absent, empty, - "clearing the list is not the same as not overriding it" - ); - assert_ne!(empty, one); - assert_ne!(one, two); - - // Order is part of the value: the list is the operator's own, not an - // accumulation of independent rows, so a reorder is a real edit. - let reordered = policy_fingerprint(Some(&fp_entry( - Some("auto"), - Some(vec!["filing.submit", "payment.send"]), - ))); - assert_ne!(two, reordered); - - // Length is folded in, so concatenation cannot collide. - let split = policy_fingerprint(Some(&fp_entry(Some("auto"), Some(vec!["a", "b"])))); - let joined = policy_fingerprint(Some(&fp_entry(Some("auto"), Some(vec!["ab"])))); - assert_ne!(split, joined); - } - - /// Issue #661 / L5: an overlay teammate's own `tools` grant flows into the - /// manifest shape `build_agent` consumes, and is INTERSECTED with the company - /// allow-list — narrow-only, never a widen. An empty grant is the standard - /// company-wide grant, exactly as the pre-L5 hardcoded empty was. - #[test] - fn overlay_agent_to_manifest_carries_the_tool_grant() { - let allow = vec!["docs.*".to_string(), "web".to_string()]; - - // A scoped overlay teammate: the grant is carried, then narrowed to what - // the company already allows. `payment.send` is NOT in `allow`, so the - // overlay cannot escalate to it — the security invariant. - let scoped = OverlayAgent { - id: "scoped".into(), - name: "Scoped".into(), - role: "Researcher".into(), - description: None, - tools: vec!["docs.*".into(), "payment.send".into()], - }; - let manifest = overlay_agent_to_manifest(&scoped); - assert_eq!( - manifest.tools, - vec!["docs.*".to_string(), "payment.send".to_string()], - "the overlay's own grant must reach the manifest shape" - ); - assert_eq!( - agent_effective_grants(&allow, &manifest.tools), - vec!["docs.*".to_string()], - "narrow-only: the un-allowed `payment.send` is intersected out" - ); - - // An empty overlay grant is the standard company-wide grant, unchanged. - let standard = OverlayAgent { - id: "std".into(), - name: "Std".into(), - role: "Generalist".into(), - description: None, - tools: Vec::new(), - }; - let manifest = overlay_agent_to_manifest(&standard); - assert!(manifest.tools.is_empty()); - assert_eq!( - agent_effective_grants(&allow, &manifest.tools), - allow, - "an empty grant falls back to the full company allow-list" - ); - } - - /// Issue #1105: the overlay's display name is the only place the operator's - /// chosen name exists, and the console shows it on the DM header, subtitle - /// and composer. Dropping it here left the persona framed from the role - /// alone, so the teammate denied being the person on its own header. - #[test] - fn overlay_agent_to_manifest_carries_the_display_name() { - let overlay = OverlayAgent { - id: "alex".into(), - name: "Alex".into(), - role: "Content Writer".into(), - description: None, - tools: Vec::new(), - }; - - let manifest = overlay_agent_to_manifest(&overlay); - assert_eq!(manifest.name.as_deref(), Some("Alex")); - // And it reaches the one place it has to: the persona the model reads. - let persona = crate::company::prompt::persona_prompt("Acme", &manifest); - assert!( - persona.contains("You are Alex, the Content Writer at Acme"), - "{persona}" - ); - } - - /// Issue #661 / L5: a grant edit changes the roster the harness must build, so - /// it has to move the overlay fingerprint — otherwise a re-grant would - /// persist, render as applied, and be silently ignored until the process - /// restarted, the same staleness the tier/skill fingerprints guard against. - #[test] - fn overlay_fingerprint_moves_on_a_tools_only_edit() { - let one = |tools: Vec| { - vec![OverlayAgent { - id: "a".into(), - name: "A".into(), - role: "r".into(), - description: None, - tools, - }] - }; - let standard = one(Vec::new()); - let scoped = one(vec!["docs.*".into()]); - let scoped_more = one(vec!["docs.*".into(), "email".into()]); - - assert_ne!( - overlay_fingerprint(&standard), - overlay_fingerprint(&scoped), - "adding a grant must move the fingerprint or the re-grant is ignored until restart" - ); - assert_ne!( - overlay_fingerprint(&scoped), - overlay_fingerprint(&scoped_more), - "widening the grant list must move it too" - ); - // Identical grants → identical fingerprint (no spurious rebuild). - assert_eq!( - overlay_fingerprint(&scoped), - overlay_fingerprint(&one(vec!["docs.*".into()])) - ); - } - - /// Attribution is deliberately NOT hashed. - /// - /// Re-setting the same tier writes a fresh `set_by`/`at_millis`. If those - /// moved the fingerprint, every such save would rebuild the roster and drop - /// live agent sessions for a change no agent can observe — the same reason - /// `budget_fingerprint` omits them. - #[test] - fn re_setting_the_same_tier_does_not_rebuild_the_roster() { - use crate::ports::types::{Actor, ActorKind}; - let first = fp_entry(Some("auto"), Some(vec!["payment.send"])); - let second = PolicyOverride { - set_by: Actor { - kind: ActorKind::User, - id: "a-different-admin".to_string(), - }, - at_millis: 1_900_000_000_000, - ..first.clone() - }; - assert_eq!( - policy_fingerprint(Some(&first)), - policy_fingerprint(Some(&second)), - "attribution must not move the fingerprint" - ); - } - - /// In-memory `ContextStore` so `OcMemory` has somewhere to land. - #[derive(Default)] - struct MockContext { - chunks: StdMutex>, - } - - #[async_trait] - impl ContextStore for MockContext { - async fn put(&self, _id: &CompanyId, chunk: ContextChunk) -> crate::Result { - let mut guard = self.chunks.lock().unwrap(); - let addr = ChunkAddr::new(format!("addr-{}", guard.len())); - guard.push((addr.clone(), chunk)); - Ok(addr) - } - async fn list(&self, _id: &CompanyId, prefix: &str) -> crate::Result> { - let guard = self.chunks.lock().unwrap(); - Ok(guard - .iter() - .filter(|(_, c)| c.label.starts_with(prefix)) - .map(|(addr, c)| ChunkMeta { - addr: addr.clone(), - label: c.label.clone(), - len: c.body.len(), - // The mock does not model store time; these tests exercise - // the harness, not the Brain's freshness stat. - stored_at_millis: 0, - }) - .collect()) - } - async fn peek( - &self, - _id: &CompanyId, - addr: &ChunkAddr, - _range: Option>, - ) -> crate::Result { - let guard = self.chunks.lock().unwrap(); - Ok(guard - .iter() - .find(|(a, _)| a == addr) - .map(|(_, c)| c.body.clone()) - .unwrap_or_default()) - } - async fn search( - &self, - _id: &CompanyId, - query: &str, - limit: usize, - ) -> crate::Result> { - let guard = self.chunks.lock().unwrap(); - Ok(guard - .iter() - .filter(|(_, c)| c.body.contains(query)) - .take(limit) - .map(|(addr, c)| ChunkHit { - addr: addr.clone(), - snippet: c.body.clone(), - score: 1.0, - }) - .collect()) - } - } - - /// `CompanyStore` that records what the cost hook appends. - #[derive(Default)] - struct RecordingStore { - ledger: StdMutex>, - } - - #[async_trait] - impl CompanyStore for RecordingStore { - async fn load(&self, _id: &CompanyId) -> crate::Result> { - Ok(None) - } - async fn save(&self, _record: &CompanyRecord) -> crate::Result<()> { - Ok(()) - } - async fn list(&self) -> crate::Result> { - Ok(Vec::new()) - } - async fn append_ledger(&self, _id: &CompanyId, entry: LedgerEntry) -> crate::Result<()> { - self.ledger.lock().unwrap().push(entry); - Ok(()) - } - } - - /// Records usage samples so a zero-usage turn can be asserted inert. - #[derive(Default)] - struct RecordingMeter { - samples: StdMutex>, - } - - #[async_trait] - impl UsageMeter for RecordingMeter { - async fn record(&self, _company: &CompanyId, sample: &UsageSample) -> crate::Result<()> { - self.samples.lock().unwrap().push(sample.clone()); - Ok(()) - } - /// Honours `since_millis`, per the port contract ("every sample at or - /// after `since_millis`"). The per-agent daily cap (issue #304) is a - /// windowed read, so a double that returned everything regardless would - /// make the day-rollover test pass against any boundary the code - /// computed — including none at all. - async fn query(&self, _company: &CompanyId, since: u64) -> crate::Result> { - Ok(self - .samples - .lock() - .unwrap() - .iter() - .filter(|sample| sample.at_millis >= since) - .cloned() - .collect()) - } - } - - /// A meter whose reads always fail — for the dispatch gate's fail-open pin. - struct FailingMeter; - - #[async_trait] - impl UsageMeter for FailingMeter { - async fn record(&self, _company: &CompanyId, _sample: &UsageSample) -> crate::Result<()> { - Ok(()) - } - async fn query( - &self, - _company: &CompanyId, - _since: u64, - ) -> crate::Result> { - Err(OpenCompanyError::Store("meter unavailable".into())) - } - } - - fn manifest() -> CompanyManifest { - toml::from_str( - r#" -[company] -name = "Acme" - -[policy] -mode = "full" - -[[agent]] -id = "ceo" -role = "Chief Executive" -description = "Sets direction." - -[[agent]] -id = "engineer" -role = "Engineer" -description = "Builds the product." -"#, - ) - .expect("valid manifest") - } - - fn record() -> CompanyRecord { - CompanyRecord { - id: CompanyId::new("acme"), - manifest: manifest(), - ledger: Vec::new(), - lifecycle: "running".to_string(), - overlay_agents: Vec::new(), - overlay_desk_members: Vec::new(), - overlay_desk_order: Vec::new(), - overlay_desks: Vec::new(), - overlay_workflows: Vec::new(), - overlay_budgets: Vec::new(), - overlay_policy: None, - overlay_desk_tools: Default::default(), - disabled_workflows: Vec::new(), - template_provenance: None, - setup: None, - } - } - - struct Fixture { - deps: HarnessDeps, - store: Arc, - meter: Arc, - _dir: tempfile::TempDir, - } - - fn fixture() -> Fixture { - let dir = tempfile::tempdir().expect("tempdir"); - let store = Arc::new(RecordingStore::default()); - let meter = Arc::new(RecordingMeter::default()); - Fixture { - deps: HarnessDeps { - ledgers: None, - ledger_registry: Default::default(), - provider: Arc::new(MockProvider::new("mock: ")), - provider_slug: "mock".to_string(), - context: Arc::new(MockContext::default()), - store: store.clone(), - meter: Some(meter.clone()), - workspace_root: dir.path().to_path_buf(), - workspace_git_enabled: false, - audit_root: dir.path().to_path_buf(), - model_override: None, - tasks: None, - artifacts: None, - skills: None, - skills_source_dir: None, - skills_registry: std::sync::Arc::from([]), - default_mcp_servers: Vec::new(), - mcp_servers: Vec::new(), - facts: None, - events: None, - delegations: DelegationQueue::default(), - workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), - mcp_failures: McpFailureQueue::default(), - pending_publishes: crate::harness::publish::PendingPublishQueue::default(), - workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), - run_outputs: crate::harness::orchestrator::RunOutputCache::default(), - run_output_store: None, - workflow_revisions: None, - approval_requests: ApprovalRequestQueue::default(), - secrets: None, - web_allowed_domains: Vec::new(), - capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, - workflow_source_dir: None, - plan: None, - media: None, - composio: None, - #[cfg(feature = "chargebee")] - chargebee: None, - #[cfg(feature = "paypal")] - paypal: None, - hosting: None, - steer: crate::company::steer::InflightRegistry::default(), - run_supervisor: crate::runtime::RunSupervisor::default(), - delivery: None, - search: None, - workspace: None, - repos: None, - repo_bindings: Vec::new(), - checkouts: crate::harness::repo::CheckoutLedger::default(), - }, - store, - meter, - _dir: dir, - } - } - - #[tokio::test] - async fn roster_builds_every_manifest_agent() { - let fx = fixture(); - let roster = - build_roster(&record(), &fx.deps, &[], &HashMap::new()).expect("roster builds"); - let ids: Vec<_> = roster.iter().map(|a| a.agent_id.as_str()).collect(); - assert_eq!(ids, vec!["ceo", "engineer"]); - assert_eq!(roster[0].role, "Chief Executive"); - } - - /// Context routing: the resolution that feeds a persona, and the fingerprint - /// that decides whether an edit reaches the next turn. - mod routed_context { - use super::*; - use crate::ports::workspace::{NodeKind, WorkspaceNode, WorkspaceOrigin}; - - fn docs(entries: &[(&str, &[(&str, &str)])]) -> HashMap> { - entries - .iter() - .map(|(agent, documents)| { - ( - (*agent).to_string(), - documents - .iter() - .map(|(p, b)| ((*p).to_string(), (*b).to_string())) - .collect(), - ) - }) - .collect() - } - - /// The property the whole axis exists for. The routing table is manifest - /// data and does not move when an operator edits a note, so a - /// name-only hash would leave the edit invisible until a restart. - #[test] - fn the_fingerprint_moves_when_a_documents_body_changes() { - let before = routed_context_fingerprint(&docs(&[("ceo", &[("BRIEF.md", "old")])])); - let after = routed_context_fingerprint(&docs(&[("ceo", &[("BRIEF.md", "new")])])); - assert_ne!( - before, after, - "an edited routed note must rebuild the roster" - ); - } - - /// A `HashMap` has no order, so an order-sensitive hash would drop every - /// live agent session on a rebuild that changed nothing. - #[test] - fn the_fingerprint_is_stable_across_map_iteration_order() { - let one = docs(&[ - ("ceo", &[("BRIEF.md", "b")]), - ("engineer", &[("CLAIMS.md", "c")]), - ]); - let two = docs(&[ - ("engineer", &[("CLAIMS.md", "c")]), - ("ceo", &[("BRIEF.md", "b")]), - ]); - assert_eq!( - routed_context_fingerprint(&one), - routed_context_fingerprint(&two) - ); - } - - /// Renaming a document is a real change even when its text is identical: - /// the persona quotes the path as the section heading. - #[test] - fn the_fingerprint_moves_when_a_document_is_renamed() { - let before = routed_context_fingerprint(&docs(&[("ceo", &[("BRIEF.md", "same")])])); - let after = routed_context_fingerprint(&docs(&[("ceo", &[("GOAL.md", "same")])])); - assert_ne!(before, after); - } - - /// A company with no workspace store keeps a stable fingerprint and never - /// rebuilds on this axis — the pre-routing behaviour exactly. - #[tokio::test] - async fn no_workspace_store_resolves_to_nothing() { - let fx = fixture(); - assert!(fx.deps.workspace.is_none(), "fixture has no store wired"); - - let pool = HarnessPool::new(); - let routed = pool.resolve_routed_context(&record(), &fx.deps, &[]).await; - assert!(routed.is_empty(), "{routed:?}"); - assert_eq!( - routed_context_fingerprint(&routed), - routed_context_fingerprint(&HashMap::new()), - "a company that routes nothing must not rebuild on this axis" - ); - } - - /// The real path: a routed document that exists in the tree is read and - /// keyed to the agent whose manifest asked for it. - #[tokio::test] - async fn a_routed_document_is_resolved_per_agent() { - let dir = tempfile::tempdir().expect("tempdir"); - let ws: Arc = - Arc::new(crate::store::FsOps::new(dir.path())); - let company = CompanyId::new("acme"); - ws.create( - &company, - &WorkspaceNode { - id: "n-brief".to_string(), - name: "BRIEF.md".to_string(), - kind: NodeKind::File, - parent_id: None, - updated_at_millis: 1, - created_by: WorkspaceOrigin::Operator, - updated_by: WorkspaceOrigin::Operator, - mime: None, - size: None, - sha256: None, - }, - Some("What the company established."), - ) - .await - .expect("create"); - - let mut fx = fixture(); - fx.deps.workspace = Some(ws); - - let pool = HarnessPool::new(); - let routed = pool.resolve_routed_context(&record(), &fx.deps, &[]).await; - - // Both fixture agents default to the `reasoning` row, which routes - // BRIEF — so both resolve it, and neither invents the notes that do - // not exist in the tree. - for agent in ["ceo", "engineer"] { - let documents = routed - .get(agent) - .unwrap_or_else(|| panic!("no routed documents for {agent}: {routed:?}")); - assert_eq!( - documents, - &vec![( - "BRIEF.md".to_string(), - "What the company established.".to_string() - )], - "{agent}" - ); - } - } - } - - /// The roster builds end-to-end with the skill read surface wired: the - /// effective set materializes, the read tools build, and the catalogue folds - /// into the persona — all without error — and the scratch tree lands under - /// the agent's workspace root. - #[tokio::test] - async fn roster_builds_with_skill_surface_wired() { - let dir = tempfile::tempdir().expect("tempdir"); - let source = tempfile::tempdir().expect("source"); - let skill_dir = source.path().join("skills").join("web-research"); - std::fs::create_dir_all(&skill_dir).unwrap(); - std::fs::write( - skill_dir.join("SKILL.md"), - "---\nname: Web Research\ndescription: Answer a question\n---\n\n# Web Research\n", - ) - .unwrap(); - - let deps = HarnessDeps { - ledgers: None, - ledger_registry: Default::default(), - provider: Arc::new(MockProvider::new("mock: ")), - provider_slug: "mock".to_string(), - context: Arc::new(MockContext::default()), - store: Arc::new(RecordingStore::default()), - meter: None, - workspace_root: dir.path().to_path_buf(), - workspace_git_enabled: false, - audit_root: dir.path().to_path_buf(), - model_override: None, - tasks: None, - artifacts: None, - skills: None, - skills_source_dir: Some(source.path().to_path_buf()), - skills_registry: std::sync::Arc::from([]), - default_mcp_servers: Vec::new(), - mcp_servers: Vec::new(), - facts: None, - events: None, - delegations: DelegationQueue::default(), - workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), - mcp_failures: McpFailureQueue::default(), - pending_publishes: crate::harness::publish::PendingPublishQueue::default(), - workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), - run_outputs: crate::harness::orchestrator::RunOutputCache::default(), - run_output_store: None, - workflow_revisions: None, - approval_requests: ApprovalRequestQueue::default(), - secrets: None, - web_allowed_domains: Vec::new(), - capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, - workflow_source_dir: None, - plan: None, - media: None, - composio: None, - #[cfg(feature = "chargebee")] - chargebee: None, - #[cfg(feature = "paypal")] - paypal: None, - hosting: None, - steer: crate::company::steer::InflightRegistry::default(), - run_supervisor: crate::runtime::RunSupervisor::default(), - delivery: None, - search: None, - workspace: None, - repos: None, - repo_bindings: Vec::new(), - checkouts: crate::harness::repo::CheckoutLedger::default(), - }; - - let roster = build_roster(&record(), &deps, &[], &HashMap::new()) - .expect("roster builds with skills"); - assert_eq!(roster.len(), 2); - // The scratch skill tree was materialized for the first roster agent. - assert!( - dir.path() - .join("acme") - .join("ceo") - .join("skill-catalog") - .join("skills") - .join("web-research") - .join("SKILL.md") - .is_file(), - "the effective skill bundle should be materialized under the agent workspace" - ); - } - - /// Issue #71 — an operator/orchestrator-added overlay teammate is promoted - /// into a real, addressable roster agent (not just a console row). - #[tokio::test] - async fn overlay_agent_is_built_as_a_real_roster_agent() { - let fx = fixture(); - let mut rec = record(); - rec.overlay_agents.push(OverlayAgent { - id: "growth".into(), - name: "Jamie".into(), - role: "Growth Lead".into(), - description: Some("Owns acquisition experiments.".into()), - tools: Vec::new(), - }); - - let roster = build_roster(&rec, &fx.deps, &[], &HashMap::new()).expect("roster builds"); - let ids: Vec<_> = roster.iter().map(|a| a.agent_id.as_str()).collect(); - assert_eq!(ids, vec!["ceo", "engineer", "growth"], "got {ids:?}"); - let overlay_agent = roster - .iter() - .find(|a| a.agent_id == "growth") - .expect("overlay teammate present in roster"); - assert_eq!(overlay_agent.role, "Growth Lead"); - } - - /// A manifest agent always wins an id collision with an overlay teammate — - /// the version-controlled roster is authoritative. - #[tokio::test] - async fn overlay_agent_id_colliding_with_manifest_agent_is_skipped() { - let fx = fixture(); - let mut rec = record(); - rec.overlay_agents.push(OverlayAgent { - id: "ceo".into(), - name: "Impostor".into(), - role: "Shadow CEO".into(), - description: None, - tools: Vec::new(), - }); - - let roster = build_roster(&rec, &fx.deps, &[], &HashMap::new()).expect("roster builds"); - let ids: Vec<_> = roster.iter().map(|a| a.agent_id.as_str()).collect(); - assert_eq!( - ids, - vec!["ceo", "engineer"], - "the manifest agent wins the id collision, not a duplicate" - ); - assert_eq!( - roster[0].role, "Chief Executive", - "the manifest role survives, not the overlay's" - ); - } - - /// Issue #686, end to end: the orchestrator adds a teammate whose display - /// name slugs onto a **manifest** agent's id, and the teammate still shows - /// up in the built roster. - /// - /// This is the failure the suffix exists to prevent, and it only became - /// reachable when ids started coming from names. `add_agent`'s duplicate - /// guard compares overlay *names*, so "Engineer" sails past it; an - /// unsuffixed `engineer` would then be skipped by - /// [`build_roster`](super::build_roster) as already claimed by the manifest - /// — saved to the record, never materialised, no error anywhere. - #[tokio::test] - async fn a_tool_added_teammate_colliding_with_a_manifest_id_still_joins_the_roster() { - use openhuman_core::openhuman::tools::Tool; - - use crate::harness::orchestrator::unscoped_add_agent; - - /// A `CompanyStore` that actually holds the record, unlike - /// `RecordingStore` — `add_agent` has to load what it saves. - struct SeededStore(StdMutex); - - #[async_trait] - impl CompanyStore for SeededStore { - async fn load(&self, _id: &CompanyId) -> crate::Result> { - Ok(Some(self.0.lock().unwrap().clone())) - } - async fn save(&self, record: &CompanyRecord) -> crate::Result<()> { - *self.0.lock().unwrap() = record.clone(); - Ok(()) - } - async fn list(&self) -> crate::Result> { - Ok(Vec::new()) - } - async fn append_ledger( - &self, - _id: &CompanyId, - _entry: LedgerEntry, - ) -> crate::Result<()> { - Ok(()) - } - } - - let fx = fixture(); - let company = CompanyId::new("acme"); - let store = Arc::new(SeededStore(StdMutex::new(record()))); - let tool = unscoped_add_agent(company.clone(), store.clone()); - - let result = tool - .execute(serde_json::json!({ "name": "Engineer", "role": "Platform" })) - .await - .expect("execute"); - assert!( - !result.is_error, - "the name guard compares overlay names only" - ); - assert!( - result.text().contains("engineer_2"), - "the orchestrator has to learn the id it can address: {}", - result.text() - ); - - let saved = store.load(&company).await.unwrap().expect("record"); - assert_eq!(saved.overlay_agents[0].id, "engineer_2"); - - let roster = build_roster(&saved, &fx.deps, &[], &HashMap::new()).expect("roster builds"); - let ids: Vec<_> = roster.iter().map(|a| a.agent_id.as_str()).collect(); - assert_eq!( - ids, - vec!["ceo", "engineer", "engineer_2"], - "a suffixed id materialises; an unsuffixed one would vanish here" - ); - } - - /// Issue #551: a roster rebuild writes nothing to the workspace. - /// - /// This used to be the feature's second provisioning seam — a teammate - /// added at runtime (a manifest edit, the console's `add_member`, the - /// orchestrator's `add_agent`) reaches the harness as a moved overlay - /// fingerprint, and the folder was minted here. A member folder is no - /// longer a function of the roster, so joining one is no longer an event - /// the tree records: the folder appears when the teammate first produces - /// something, and the two system roots come from boot. - /// - /// Pinned as a test because a rebuild that quietly resumed writing would - /// re-fill the tree with empty folders for teammates who have done nothing - /// — exactly the noise this change removed. - #[tokio::test] - async fn a_roster_rebuild_writes_nothing_to_the_workspace() { - let dir = tempfile::tempdir().expect("tempdir"); - let ws: Arc = - Arc::new(crate::store::FsOps::new(dir.path())); - let mut fx = fixture(); - fx.deps.workspace = Some(ws.clone()); - - let mut rec = record(); - let pool = HarnessPool::new(); - pool.ensure(&rec, &fx.deps).await.expect("first ensure"); - assert!( - ws.is_empty(&rec.id).await.expect("is_empty"), - "the roster build touched the workspace" - ); - - // The runtime-added teammate. The overlay fingerprint moves, so this - // `ensure` takes the rebuild path rather than the cached fast path. - rec.overlay_agents.push(OverlayAgent { - id: "designer".into(), - name: "Dana".into(), - role: "Designer".into(), - description: None, - tools: Vec::new(), - }); - pool.ensure(&rec, &fx.deps).await.expect("second ensure"); - - assert!( - ws.is_empty(&rec.id).await.expect("is_empty"), - "the rebuild minted a folder for a teammate that has produced nothing" - ); - - // …and the folder the teammate *does* get is the one it earns by - // producing something, minted through the lazy seam instead. - let minted = crate::company::workspace_scaffold::ensure_agent_folder( - ws.as_ref(), - &rec.id, - "designer", - ) - .await - .expect("mint"); - let tree = ws.tree(&rec.id).await.expect("tree"); - let mut names: Vec<&str> = tree.iter().map(|n| n.name.as_str()).collect(); - names.sort_unstable(); - assert_eq!(names, vec!["Agents", "designer"]); - assert_eq!( - tree.iter().find(|n| n.id == minted).unwrap().created_by, - crate::ports::WorkspaceOrigin::Agent { - id: "designer".to_string() - }, - ); - } - - #[tokio::test] - async fn run_executes_a_turn_on_the_openhuman_runtime() { - let fx = fixture(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("ensure"); - - let reply = pool - .run(&rec.id, "ceo", "hello-marker", &fx.deps, None) - .await - .expect("turn runs") - .reply; - - assert!( - reply.contains("hello-marker"), - "reply should echo the prompt through the agent: {reply:?}" - ); - } - - #[tokio::test] - async fn run_stores_outcomes_and_injects_them_into_later_turns() { - let fx = fixture(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("ensure"); - - // Cold store: nothing to inject on the first turn. - let first = pool - .run(&rec.id, "ceo", "alpha task", &fx.deps, None) - .await - .expect("first turn") - .reply; - assert!( - !first.contains("Relevant prior work"), - "a cold turn injects nothing: {first:?}" - ); - - // The outcome was written back under the task-outcome prefix. - let stored = fx - .deps - .context - .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) - .await - .unwrap(); - assert_eq!(stored.len(), 1, "the first turn stores its outcome"); - - // Second turn: the prior outcome (its body contains "alpha") is - // retrieved and injected, so the agent sees the preamble. - let second = pool - .run(&rec.id, "ceo", "alpha", &fx.deps, None) - .await - .expect("second turn") - .reply; - assert!( - second.contains("Relevant prior work"), - "the second turn injects the retrieved outcome: {second:?}" - ); - - let stored = fx - .deps - .context - .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) - .await - .unwrap(); - assert_eq!(stored.len(), 2, "the second turn stores its outcome too"); - } - - #[tokio::test] - async fn ensure_is_idempotent() { - let fx = fixture(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("first ensure"); - pool.ensure(&rec, &fx.deps).await.expect("second ensure"); - assert_eq!(pool.resident_companies().await, 1); - } - - #[tokio::test] - async fn turns_are_serialised_and_history_survives() { - let fx = fixture(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("ensure"); - - pool.run(&rec.id, "ceo", "first", &fx.deps, None) - .await - .expect("first turn"); - let second = pool - .run(&rec.id, "ceo", "second", &fx.deps, None) - .await - .expect("second turn") - .reply; - - assert!(second.contains("second")); - } - - /// Issue #416 — a confined turn reaches the company's memory neither on the - /// way in nor on the way out. - /// - /// The control half is what makes this a test rather than an assertion of - /// absence: the SAME message on the ordinary roster path pulls the seeded - /// chunk into the prompt (the mock provider echoes what it was sent, so the - /// injection is visible in the reply), and writes the turn back. The - /// confined path does neither, from the same store, in the same test. - #[tokio::test] - async fn a_confined_turn_neither_reads_nor_writes_company_memory() { - let context = Arc::new(MockContext::default()); - let mut fx = fixture(); - fx.deps.context = context.clone(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("ensure"); - - // A prior outcome sitting in the company's memory. The mock store - // matches a chunk whose BODY contains the query, and retrieve→inject - // queries with the whole message — so a body built around the message is - // what a hit looks like here. - let question = "why did it fail"; - context - .put( - &rec.id, - ContextChunk { - label: "prior/outcome".into(), - body: format!("SECRET-PAYROLL-REVIEW: {question} on Monday"), - }, - ) - .await - .expect("seed the company's memory"); - let seeded = context.chunks.lock().unwrap().len(); - - // Control: the ordinary path injects the hit and writes the turn back. - let ordinary = pool - .run(&rec.id, "ceo", question, &fx.deps, None) - .await - .expect("the ordinary turn runs") - .reply; - assert!( - ordinary.contains("SECRET-PAYROLL-REVIEW"), - "the retrieve→inject step must be live for this test to mean anything: {ordinary}" - ); - assert!( - context.chunks.lock().unwrap().len() > seeded, - "the ordinary path writes its outcome back to company memory" - ); - - let before_confined = context.chunks.lock().unwrap().len(); - let confined = pool - .run_confined( - &rec.id, - "Acme", - question, - &fx.deps, - Some("workflow-copilot:weekly_report"), - &confine::Confinement::workflow("weekly_report"), - ) - .await - .expect("the confined turn runs") - .reply; - - assert!( - confined.contains(question), - "the confined turn still answers the question it was asked: {confined}" - ); - assert!( - !confined.contains("SECRET-PAYROLL-REVIEW"), - "a confined turn must not be handed company memory: {confined}" - ); - assert_eq!( - context.chunks.lock().unwrap().len(), - before_confined, - "a confined turn must leave nothing behind for a later turn to retrieve" - ); - } - - /// The confined agent is not on the roster, so nothing can address it: a - /// dispatch, a desk hand-off or a `chat` naming it is an unknown agent, the - /// same as any other name that is not a teammate. - #[tokio::test] - async fn the_confined_agent_is_not_addressable() { - let fx = fixture(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("ensure"); - - let err = pool - .run(&rec.id, confine::CONFINED_AGENT_ID, "hi", &fx.deps, None) - .await - .expect_err("the confined agent is not a roster agent"); - assert!( - matches!(err, OpenCompanyError::InvalidRequest(_)), - "{err:?}" - ); - } - - #[tokio::test] - async fn unknown_agent_is_invalid_request() { - let fx = fixture(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("ensure"); - - let err = pool - .run(&rec.id, "nobody", "hi", &fx.deps, None) - .await - .expect_err("unknown agent rejected"); - assert!( - matches!(err, OpenCompanyError::InvalidRequest(_)), - "{err:?}" - ); - } - - #[tokio::test] - async fn unknown_company_is_not_found() { - let fx = fixture(); - let pool = HarnessPool::new(); - let err = pool - .run(&CompanyId::new("ghost"), "ceo", "hi", &fx.deps, None) - .await - .expect_err("unknown company rejected"); - assert!( - matches!(err, OpenCompanyError::CompanyNotFound(_)), - "{err:?}" - ); - } - - // --- Workspace-ensure log edge-triggering (issue #449) ------------------- - - /// The whole transition table, exhaustively: a broken volume must produce - /// one error line and then nothing, and a recovery must be announced once. - #[test] - fn workspace_report_is_edge_triggered() { - let mut failing: HashSet<&str> = HashSet::new(); - - // First failure speaks. - assert_eq!( - workspace_report(&mut failing, &"a", true), - WorkspaceReport::Failed - ); - // Every repeat is silent — this is the flood #449 is about. - for _ in 0..100 { - assert_eq!( - workspace_report(&mut failing, &"a", true), - WorkspaceReport::StillFailing - ); - } - // Recovery speaks exactly once. - assert_eq!( - workspace_report(&mut failing, &"a", false), - WorkspaceReport::Recovered - ); - assert_eq!( - workspace_report(&mut failing, &"a", false), - WorkspaceReport::StillHealthy - ); - // A healthy agent that was never failing says nothing on its first - // attempt either — a working workspace has never been worth a line. - assert_eq!( - workspace_report(&mut failing, &"never-failed", false), - WorkspaceReport::StillHealthy - ); - // And it can fail again later: the edge re-arms. - assert_eq!( - workspace_report(&mut failing, &"a", true), - WorkspaceReport::Failed - ); - - assert!( - WorkspaceReport::StillFailing.is_silent() && WorkspaceReport::StillHealthy.is_silent(), - "only the repeats are silent" - ); - assert!( - !WorkspaceReport::Failed.is_silent() && !WorkspaceReport::Recovered.is_silent(), - "both edges must be reported" - ); - } - - /// Two agents interleaved: one failing, one healthy. Each key's edge is its - /// own — a second agent's failure must not be swallowed by the first's, and - /// a second agent's recovery must not clear the first's failure. - #[test] - fn workspace_report_tracks_each_key_separately() { - let mut failing: HashSet<&str> = HashSet::new(); - - assert_eq!( - workspace_report(&mut failing, &"ceo", true), - WorkspaceReport::Failed - ); - // A different agent failing is its own first failure, not a repeat. - assert_eq!( - workspace_report(&mut failing, &"engineer", true), - WorkspaceReport::Failed - ); - assert_eq!( - workspace_report(&mut failing, &"ceo", true), - WorkspaceReport::StillFailing - ); - // One recovers; the other stays failing and stays silent. - assert_eq!( - workspace_report(&mut failing, &"engineer", false), - WorkspaceReport::Recovered - ); - assert_eq!( - workspace_report(&mut failing, &"ceo", true), - WorkspaceReport::StillFailing - ); - assert_eq!( - workspace_report(&mut failing, &"ceo", false), - WorkspaceReport::Recovered - ); - assert!(failing.is_empty(), "a recovered key leaves no residue"); - } - - /// The real dispatch path against a workspace root that cannot hold a - /// directory, driven through [`HarnessPool::run`] rather than the helper. - /// - /// The root is pointed at a **file**, which makes `create_dir_all` fail - /// deterministically on every platform (`ENOTDIR` / its Windows equivalent) - /// without needing permission bits a CI root user would ignore. - /// - /// Asserts the reporting state, not the log text: this test binary already - /// installs a global `tracing` subscriber elsewhere - /// (`runtime::workflow_scheduler`) and asserts it wins that race, so a - /// second global capture here would make whichever test lost panic. The - /// state is what decides whether a line is emitted, so pinning it pins the - /// line count — three dispatches, one report. - #[tokio::test] - async fn a_broken_workspace_root_reports_once_across_repeated_dispatches() { - let dir = tempfile::tempdir().expect("tempdir"); - // A regular file where the workspace tree is expected. - let not_a_dir = dir.path().join("workspace-root"); - std::fs::write(¬_a_dir, b"this is a file, not a directory").unwrap(); - - let mut fx = fixture(); - fx.deps.workspace_root = not_a_dir.clone(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("ensure"); - - // Sanity: the condition really is a hard, repeatable failure. - assert!( - build::ensure_agent_workspace(¬_a_dir, &rec.id, "ceo").is_err(), - "the test root must actually be unusable, or this proves nothing" - ); - - for turn in 0..3 { - pool.run(&rec.id, "ceo", "hi", &fx.deps, None) - .await - .unwrap_or_else(|e| panic!("turn {turn} still runs without a workspace: {e:?}")); - } - - // The turns ran — a missing workspace is not fatal, which #449 does not - // change — and the failure is recorded exactly once. - let failing = pool.workspace_failures.lock().unwrap(); - assert_eq!( - failing.len(), - 1, - "one failing agent, tracked once, however many turns it takes" - ); - assert!(failing.contains(&(rec.id.clone(), "ceo".to_string()))); - drop(failing); - - // The next dispatch after the first is silent: only turn 1 spoke. - assert_eq!( - pool.note_workspace_attempt(&rec.id, "ceo", true), - WorkspaceReport::StillFailing, - "dispatches after the first must not re-emit the error" - ); - // And when the volume comes back, one line says so. - assert_eq!( - pool.note_workspace_attempt(&rec.id, "ceo", false), - WorkspaceReport::Recovered - ); - } - - /// Pins the documented inert-metering contract: until the provider reports - /// usage, a turn writes neither a ledger entry nor a usage sample. - #[tokio::test] - async fn zero_usage_turn_writes_nothing() { - let fx = fixture(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("ensure"); - pool.run(&rec.id, "ceo", "hi", &fx.deps, None) - .await - .expect("turn"); - - assert!(fx.store.ledger.lock().unwrap().is_empty()); - assert!(fx.meter.samples.lock().unwrap().is_empty()); - } - - // --- Empty-response turn wrapper ---------------------------------------- - - /// A model that plays back a scripted sequence of outcomes, one per - /// [`invoke`](ChatModel::invoke) call, so the empty-response retry wrapper can - /// be driven deterministically. `Ok("")` is the transient empty class (the - /// harness turn raises the empty-response error on a blank assistant reply); - /// `Err(_)` is a hard error. - struct ScriptedProvider { - script: StdMutex>>, - calls: std::sync::atomic::AtomicUsize, - } - - impl ScriptedProvider { - fn new(outcomes: Vec>) -> Self { - Self { - script: StdMutex::new(outcomes.into_iter().collect()), - calls: std::sync::atomic::AtomicUsize::new(0), - } - } - } - - #[async_trait] - impl ChatModel<()> for ScriptedProvider { - async fn invoke( - &self, - _state: &(), - _request: ModelRequest, - ) -> tinyagents::Result { - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - match self.script.lock().unwrap().pop_front() { - Some(Ok(reply)) => Ok(ModelResponse::assistant(reply)), - Some(Err(err)) => Err(tinyagents::TinyAgentsError::Model(err)), - None => Ok(ModelResponse::assistant("exhausted")), - } - } - } - - impl HarnessModel for ScriptedProvider { - fn telemetry_provider_id(&self) -> String { - "scripted".to_string() - } - } - - /// Build a single [`CompanyAgent`] over a scripted provider so the wrapper can - /// be exercised directly (its retry logic is the unit under test). - fn scripted_agent(outcomes: Vec>) -> (Arc, HarnessDeps) { - let dir = tempfile::tempdir().expect("tempdir"); - let deps = HarnessDeps { - ledgers: None, - ledger_registry: Default::default(), - provider: Arc::new(ScriptedProvider::new(outcomes)), - provider_slug: "scripted".to_string(), - context: Arc::new(MockContext::default()), - store: Arc::new(RecordingStore::default()), - meter: None, - workspace_root: dir.path().to_path_buf(), - workspace_git_enabled: false, - audit_root: dir.path().to_path_buf(), - model_override: None, - tasks: None, - artifacts: None, - skills: None, - skills_source_dir: None, - skills_registry: std::sync::Arc::from([]), - default_mcp_servers: Vec::new(), - mcp_servers: Vec::new(), - facts: None, - events: None, - delegations: DelegationQueue::default(), - workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), - mcp_failures: McpFailureQueue::default(), - pending_publishes: crate::harness::publish::PendingPublishQueue::default(), - workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), - run_outputs: crate::harness::orchestrator::RunOutputCache::default(), - run_output_store: None, - workflow_revisions: None, - approval_requests: ApprovalRequestQueue::default(), - secrets: None, - web_allowed_domains: Vec::new(), - capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, - workflow_source_dir: None, - plan: None, - media: None, - composio: None, - #[cfg(feature = "chargebee")] - chargebee: None, - #[cfg(feature = "paypal")] - paypal: None, - hosting: None, - steer: crate::company::steer::InflightRegistry::default(), - run_supervisor: crate::runtime::RunSupervisor::default(), - delivery: None, - search: None, - workspace: None, - repos: None, - repo_bindings: Vec::new(), - checkouts: crate::harness::repo::CheckoutLedger::default(), - }; - let roster = build_roster(&record(), &deps, &[], &HashMap::new()).expect("roster"); - // Keep the tempdir alive for the agent's workspace by leaking it into the - // test's lifetime — the process ends the test anyway. - std::mem::forget(dir); - (roster.into_iter().next().expect("one agent"), deps) - } - - /// Empty first, real reply on retry → the wrapper returns the recovered reply - /// and reports two attempts' usage (so both burnt attempts can be metered). - #[tokio::test] - async fn turn_wrapper_retries_empty_then_recovers() { - let (agent, _deps) = scripted_agent(vec![Ok(String::new()), Ok("recovered".into())]); - let (outcome, usages) = agent.run("hi").await.expect("wrapper recovers"); - assert!( - outcome.reply.contains("recovered"), - "got {:?}", - outcome.reply - ); - assert_eq!(usages.len(), 2, "both attempts' usage is returned"); - } - - /// Issue #111 retry-guard edge: when a steer already pends and the first - /// attempt is the transient empty class, the one-shot retry is SKIPPED — so a - /// cancel/pause issued before any text can't silently restart the work. The - /// steered-empty turn therefore makes EXACTLY ONE attempt. - #[tokio::test] - async fn steered_empty_turn_makes_exactly_one_attempt() { - // Attempt 1 is empty; a normal `run` would retry and consume the second - // script entry. With a steer pending, the retry must not fire. - let (agent, _deps) = scripted_agent(vec![Ok(String::new()), Ok("second".into())]); - let control = SteerControl::new(); - control.request(SteerAction::Cancel); - let (_outcome, usages) = agent - .run_with_steer("hi", Some(&control), None, None) - .await - .expect("runs"); - assert_eq!( - usages.len(), - 1, - "a steered empty turn does NOT retry — exactly one attempt" - ); - } - - // Note: the *installation* of the steer stop-hook can't be observed from the - // provider — the tinyagents adapter snapshots the hooks at turn entry and the - // provider call may run on a spawned task where the task-local isn't - // inherited. The steer mechanism is instead proven end-to-end by the - // retry-guard edge above and the `run_task` disposition matrix in - // `harness::brain::tests` (cancel / pause / redirect all take effect). - - /// Empty twice → a graceful, non-error reply (chat never shows "Couldn't - /// send" for a transient hiccup), still two attempts. - #[tokio::test] - async fn turn_wrapper_empty_twice_is_graceful() { - let (agent, _deps) = scripted_agent(vec![Ok(String::new()), Ok(String::new())]); - let (outcome, usages) = agent.run("hi").await.expect("graceful, not an Err"); - assert!( - outcome - .reply - .to_lowercase() - .contains("temporary model hiccup"), - "got {:?}", - outcome.reply - ); - assert_eq!(usages.len(), 2); - } - - /// The Empty-vs-Hard split: only the transient empty-response class is - /// retried/softened; every other error is `Hard` and propagates loudly (no - /// blanket swallow). Driven at the classifier so it's deterministic — the - /// live agent internally retries provider errors, which would make a scripted - /// "hard error" non-deterministic. - #[test] - fn transient_empty_response_is_recognised_but_hard_errors_are_not() { - let empty = anyhow::anyhow!("The model returned an empty response. Please try again."); - assert!( - is_transient_empty_response(&empty), - "empty-response is transient" - ); - - let hard = anyhow::anyhow!("daily budget exceeded for agent 'ceo'"); - assert!( - !is_transient_empty_response(&hard), - "a budget error is NOT the transient empty class — it must propagate" - ); - } - - // --- MCP-freshness ------------------------------------------------------ - - /// In-memory secret store so `ensure` can re-resolve the runtime MCP index. - #[derive(Default)] - struct MemSecrets { - map: StdMutex>, - } - - #[async_trait] - impl SecretStore for MemSecrets { - async fn get( - &self, - _c: &CompanyId, - key: &str, - ) -> crate::Result> { - Ok(self - .map - .lock() - .unwrap() - .get(key) - .map(|v| crate::ports::types::SecretValue(v.clone()))) - } - async fn set( - &self, - _c: &CompanyId, - key: &str, - value: crate::ports::types::SecretValue, - ) -> crate::Result<()> { - self.map.lock().unwrap().insert(key.to_string(), value.0); - Ok(()) - } - } - - /// A console-added MCP server reaches the agent on the NEXT `ensure`, with no - /// restart — the roster rebuilds because the effective set, re-resolved from - /// the LIVE secret store (not the boot snapshot), changed its fingerprint. - /// This is the Parallel-Search / BrowserBase freshness bug proven end-to-end, - /// and the CI guard for issue #566: the effective-MCP fingerprint is a *term* - /// of [`HarnessPool::ensure`]'s staleness check. Both directions are pinned — - /// an unchanged set holds the fingerprint (no needless rebuild), an MCP-only - /// change moves it (rebuilt in place, without a restart). A refactor that - /// drops the term makes the post-change `ensure` early-return without storing - /// the new fingerprint: the value stops moving across the mutation and the - /// `assert_ne!` fails, rather than the restart requirement quietly returning. - #[tokio::test] - async fn ensure_rebuilds_when_a_runtime_mcp_server_is_added() { - let secrets: Arc = Arc::new(MemSecrets::default()); - let dir = tempfile::tempdir().unwrap(); - let deps = HarnessDeps { - ledgers: None, - ledger_registry: Default::default(), - provider: Arc::new(MockProvider::new("mock: ")), - provider_slug: "mock".to_string(), - context: Arc::new(MockContext::default()), - store: Arc::new(RecordingStore::default()), - meter: None, - workspace_root: dir.path().to_path_buf(), - workspace_git_enabled: false, - audit_root: dir.path().to_path_buf(), - model_override: None, - tasks: None, - artifacts: None, - skills: None, - skills_source_dir: None, - skills_registry: std::sync::Arc::from([]), - default_mcp_servers: Vec::new(), - mcp_servers: Vec::new(), - facts: None, - events: None, - delegations: DelegationQueue::default(), - workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), - mcp_failures: McpFailureQueue::default(), - pending_publishes: crate::harness::publish::PendingPublishQueue::default(), - workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), - run_outputs: crate::harness::orchestrator::RunOutputCache::default(), - run_output_store: None, - workflow_revisions: None, - approval_requests: ApprovalRequestQueue::default(), - secrets: Some(secrets.clone()), - web_allowed_domains: Vec::new(), - capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, - workflow_source_dir: None, - plan: None, - media: None, - composio: None, - #[cfg(feature = "chargebee")] - chargebee: None, - #[cfg(feature = "paypal")] - paypal: None, - hosting: None, - steer: crate::company::steer::InflightRegistry::default(), - run_supervisor: crate::runtime::RunSupervisor::default(), - delivery: None, - search: None, - workspace: None, - repos: None, - repo_bindings: Vec::new(), - checkouts: crate::harness::repo::CheckoutLedger::default(), - }; - let pool = HarnessPool::new(); - let rec = record(); - - pool.ensure(&rec, &deps).await.expect("first ensure"); - let before = pool - .mcp_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - - // Stability direction: with no axis changed, a redundant `ensure` is a - // no-op — the gate reuses the cached roster and the fingerprint holds, so - // the change-direction assertion below can't pass by coincidence. - pool.ensure(&rec, &deps).await.expect("redundant ensure"); - assert_eq!( - pool.mcp_fingerprint_of(&rec.id).await, - Some(before), - "an unchanged MCP set must not move the fingerprint" - ); - - // Console-add a runtime MCP server directly into the live secret store. - crate::company::mcp::save_runtime_index( - &rec.id, - secrets.as_ref(), - &[crate::company::McpServer { - name: "browserbase".into(), - endpoint: "https://api.browserbase.com/mcp".into(), - description: None, - command: None, - allowed_tools: Vec::new(), - disallowed_tools: Vec::new(), - timeout_secs: 30, - enabled: true, - auth_secret: None, - }], - ) - .await - .unwrap(); - - // Change direction: the next ensure re-resolves from the live store → - // fingerprint changes → roster rebuilt, so the new server reaches the - // agent without a restart. - pool.ensure(&rec, &deps).await.expect("post-add ensure"); - let after = pool - .mcp_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert_ne!( - before, after, - "an MCP-only change must move the staleness fingerprint (issue #566)" - ); - assert_eq!( - pool.resident_companies().await, - 1, - "same company, rebuilt in place — not a new residency" - ); - - // Stability after the change too: a further ensure with no new change is - // a no-op and the fingerprint holds at its post-change value. - pool.ensure(&rec, &deps).await.expect("final no-op ensure"); - assert_eq!(pool.mcp_fingerprint_of(&rec.id).await, Some(after)); - } - - // --- Bound-repository freshness (issue #245) ---------------------------- - - /// A bind, a credential **rotation** and a revoke each rebuild the roster on - /// the company's next turn, with no restart. - /// - /// All three are asserted because they fail differently, and the middle one - /// is the reason the fingerprint is over `(key, token_fingerprint, - /// branches)` rather than over the set of keys. A rotation changes nothing - /// about *which* repositories exist; a revoke blanks a credential while the - /// key survives for the moment before the entry is dropped. A roster keyed - /// on the key set alone holds through both, and an agent is left holding a - /// tool over a binding that can no longer fetch. - /// - /// The index is written straight into the live secret store rather than - /// through `bind`, because what is under test is the *staleness gate*, and - /// binding for real would drag a `git` fixture and a network-shaped code - /// path into a test about a hash. - #[tokio::test] - async fn ensure_rebuilds_when_a_repository_is_bound_rotated_or_revoked() { - use crate::runtime::repo_manager::types::RepoBinding; - - let secrets: Arc = Arc::new(MemSecrets::default()); - let dir = tempfile::tempdir().unwrap(); - let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); - deps.secrets = Some(secrets.clone()); - deps.repos = Some(Arc::new(crate::runtime::RepoManager::new( - CompanyId::new("acme"), - dir.path().join("repos"), - secrets.clone(), - ))); - - // The grant is what opens this axis at all: a company that does not - // explicitly grant `repo` never reads the index, so its fingerprint can - // never move. That is the fast path every other company stays on. - let mut rec = record(); - rec.manifest.tools.allow = vec!["repo".to_string()]; - - let pool = HarnessPool::new(); - let write_index = |bindings: Vec| { - let secrets = secrets.clone(); - async move { - let json = serde_json::to_string(&serde_json::json!({ "bindings": bindings })) - .expect("index json"); - secrets - .set( - &CompanyId::new("acme"), - crate::runtime::repo_manager::REPO_INDEX_KEY, - crate::ports::types::SecretValue(json), - ) - .await - .expect("write index"); - } - }; - let binding = |fingerprint: &str| RepoBinding { - key: "acme-widgets-000000000000".to_string(), - url: "https://github.com/acme/widgets".to_string(), - owner: "acme".to_string(), - repo: "widgets".to_string(), - branches: vec!["main".to_string()], - token_fingerprint: fingerprint.to_string(), - last_fetched_millis: None, - size_bytes: 0, - bound_at_millis: 1, - can_push: None, - }; - - pool.ensure(&rec, &deps).await.expect("first ensure"); - let empty = pool - .repo_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - - // Stability first, so every change assertion below cannot pass by - // coincidence. - pool.ensure(&rec, &deps).await.expect("redundant ensure"); - assert_eq!( - pool.repo_fingerprint_of(&rec.id).await, - Some(empty), - "an unchanged binding set must not move the fingerprint" - ); - - // Bind. - write_index(vec![binding("0f1e2d3c4b5a")]).await; - pool.ensure(&rec, &deps).await.expect("post-bind ensure"); - let bound = pool - .repo_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - assert_ne!(empty, bound, "a bind must move the staleness fingerprint"); - - // Rotate: same repository, same branches, new credential. - write_index(vec![binding("aaaaaaaaaaaa")]).await; - pool.ensure(&rec, &deps).await.expect("post-rotate ensure"); - let rotated = pool - .repo_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - assert_ne!( - bound, rotated, - "a credential rotation must move the fingerprint even though the \ - repository set is identical" - ); - - // Revoke. - write_index(Vec::new()).await; - pool.ensure(&rec, &deps).await.expect("post-revoke ensure"); - let revoked = pool - .repo_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - assert_ne!(rotated, revoked, "a revoke must move the fingerprint"); - assert_eq!(revoked, empty, "and must land back on the empty set"); - assert_eq!( - pool.resident_companies().await, - 1, - "same company, rebuilt in place — not a new residency" - ); - } - - /// A company that does not explicitly grant `repo` never reads the binding - /// index, so this axis is inert for it — the fast path every company that - /// does not use the feature stays on. - #[tokio::test] - async fn a_company_without_the_repo_grant_never_moves_on_this_axis() { - use crate::runtime::repo_manager::types::RepoBinding; - - let secrets: Arc = Arc::new(MemSecrets::default()); - let dir = tempfile::tempdir().unwrap(); - let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); - deps.secrets = Some(secrets.clone()); - deps.repos = Some(Arc::new(crate::runtime::RepoManager::new( - CompanyId::new("acme"), - dir.path().join("repos"), - secrets.clone(), - ))); - - // A wildcard, deliberately: `*` does not confer `repo`, so even a - // broadly-permissioned company stays off this axis. - let mut rec = record(); - rec.manifest.tools.allow = vec!["*".to_string()]; - - let pool = HarnessPool::new(); - pool.ensure(&rec, &deps).await.expect("first ensure"); - let before = pool - .repo_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - - let json = serde_json::to_string(&serde_json::json!({ - "bindings": [RepoBinding { - key: "acme-widgets-000000000000".to_string(), - url: "https://github.com/acme/widgets".to_string(), - owner: "acme".to_string(), - repo: "widgets".to_string(), - branches: vec!["main".to_string()], - token_fingerprint: "0f1e2d3c4b5a".to_string(), - last_fetched_millis: None, - size_bytes: 0, - bound_at_millis: 1, - can_push: None, - }] - })) - .unwrap(); - secrets - .set( - &CompanyId::new("acme"), - crate::runtime::repo_manager::REPO_INDEX_KEY, - crate::ports::types::SecretValue(json), - ) - .await - .unwrap(); - - pool.ensure(&rec, &deps).await.expect("post-bind ensure"); - assert_eq!( - pool.repo_fingerprint_of(&rec.id).await, - Some(before), - "an ungranted company must not read the index, let alone rebuild on it" - ); - } - - // --- Billing-credential freshness (issues #788, #789) ------------------- - - /// Saving or rotating a key in Settings → Billing must reach the agent on - /// its next turn. - /// - /// The fingerprint is the observable that makes "no restart" testable: a - /// credential that fails to move it leaves the roster cached, and the agent - /// keeps authenticating with the old key — or holds no billing tools at all - /// — until the process restarts. That failure is invisible from the tool - /// list alone, which is why this asserts the fingerprint directly. - #[tokio::test] - #[cfg(feature = "chargebee")] - async fn ensure_rebuilds_when_a_chargebee_credential_is_saved_or_rotated() { - use crate::chargebee::types::{API_KEY_SECRET, SITE_SECRET}; - - let secrets: Arc = Arc::new(MemSecrets::default()); - let dir = tempfile::tempdir().unwrap(); - let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); - deps.secrets = Some(secrets.clone()); - - // The explicit grant is what opens this axis. A `*` wildcard does not - // confer it — see the module docs. - let mut rec = record(); - rec.manifest.tools.allow = vec!["chargebee".to_string()]; - - let write = |key: &'static str, value: &'static str| { - let secrets = secrets.clone(); - async move { - secrets - .set( - &CompanyId::new("acme"), - key, - crate::ports::types::SecretValue(value.to_string()), - ) - .await - .expect("write secret"); - } - }; - - let pool = HarnessPool::new(); - pool.ensure(&rec, &deps).await.expect("first ensure"); - let unset = pool - .billing_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - - // Stability first, so every change assertion below cannot pass by - // coincidence. - pool.ensure(&rec, &deps).await.expect("redundant ensure"); - assert_eq!( - pool.billing_fingerprint_of(&rec.id).await, - Some(unset), - "an unchanged credential must not move the fingerprint" - ); - - // Half a credential is not a connection, so it must not move either — - // the pair is meaningless apart. - write(SITE_SECRET, "acme-test").await; - pool.ensure(&rec, &deps).await.expect("half ensure"); - assert_eq!( - pool.billing_fingerprint_of(&rec.id).await, - Some(unset), - "a site with no key is still no connection" - ); - - // Connect. - write(API_KEY_SECRET, "cb_first").await; - pool.ensure(&rec, &deps).await.expect("post-connect ensure"); - let connected = pool - .billing_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - assert_ne!(unset, connected, "saving a credential must rebuild"); - - // Rotate: same site, new key. This is the one a fingerprint over the - // site alone would miss, leaving the agent on the revoked key. - write(API_KEY_SECRET, "cb_rotated").await; - pool.ensure(&rec, &deps).await.expect("post-rotate ensure"); - let rotated = pool - .billing_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - assert_ne!( - connected, rotated, - "a rotation must rebuild even though the site is identical" - ); - - // Disconnect. - write(API_KEY_SECRET, "").await; - pool.ensure(&rec, &deps).await.expect("post-clear ensure"); - assert_eq!( - pool.billing_fingerprint_of(&rec.id).await, - Some(unset), - "clearing the key must land back on the unconnected fingerprint" - ); - assert_eq!( - pool.resident_companies().await, - 1, - "same company, rebuilt in place — not a new residency" - ); - } - - /// A company that does not explicitly grant `chargebee` never reads the - /// billing secrets, so this axis is inert for it — and a credential sitting - /// in its store confers nothing. Fail closed, as the module docs promise. - #[tokio::test] - #[cfg(feature = "chargebee")] - async fn a_company_without_the_chargebee_grant_never_moves_on_this_axis() { - use crate::chargebee::types::{API_KEY_SECRET, SITE_SECRET}; - - let secrets: Arc = Arc::new(MemSecrets::default()); - let dir = tempfile::tempdir().unwrap(); - let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); - deps.secrets = Some(secrets.clone()); - - // A wildcard, deliberately: it must NOT confer billing. - let mut rec = record(); - rec.manifest.tools.allow = vec!["*".to_string()]; - - let pool = HarnessPool::new(); - pool.ensure(&rec, &deps).await.expect("first ensure"); - let before = pool - .billing_fingerprint_of(&rec.id) - .await - .expect("fingerprint"); - - for (key, value) in [(SITE_SECRET, "acme-test"), (API_KEY_SECRET, "cb_key")] { - secrets - .set( - &CompanyId::new("acme"), - key, - crate::ports::types::SecretValue(value.to_string()), - ) - .await - .expect("write secret"); - } - - pool.ensure(&rec, &deps).await.expect("post-write ensure"); - assert_eq!( - pool.billing_fingerprint_of(&rec.id).await, - Some(before), - "an ungranted company must not read the billing secrets, let alone rebuild on them" - ); - } - - // --- Skill-delta freshness (issue #41) ---------------------------------- - - /// An in-memory `SkillStateStore` whose delta set a test can mutate between - /// two `ensure` calls — the same way the console Skills tab authors, edits, - /// enables, or disables a skill — so the freshness gate can be observed - /// reacting with no restart. - #[derive(Default)] - struct MemSkills { - deltas: StdMutex>, - } - - #[async_trait] - impl SkillStateStore for MemSkills { - async fn list(&self, _company: &CompanyId) -> crate::Result> { - Ok(self.deltas.lock().unwrap().clone()) - } - async fn set(&self, _company: &CompanyId, state: &SkillState) -> crate::Result<()> { - let mut deltas = self.deltas.lock().unwrap(); - match deltas.iter_mut().find(|s| s.slug == state.slug) { - Some(slot) => *slot = state.clone(), - None => deltas.push(state.clone()), - } - Ok(()) - } - async fn remove(&self, _company: &CompanyId, slug: &str) -> crate::Result { - let mut deltas = self.deltas.lock().unwrap(); - let before = deltas.len(); - deltas.retain(|s| s.slug != slug); - Ok(deltas.len() != before) - } - } - - /// A valid custom-skill delta (its `custom_doc` parses, so `materialize` - /// writes it to the scratch tree). - fn custom_skill(slug: &str, enabled: bool, body: &str) -> SkillState { - SkillState { - slug: slug.to_string(), - enabled, - source: crate::ports::skills_state::SkillSource::Custom, - custom_doc: Some(body.to_string()), - } - } - - const STANDUP_MD: &str = - "---\nname: Standup Digest\ndescription: Summarize the standup\n---\n\n# Standup Digest\n"; - - /// The scratch path a materialized skill lands at for the first roster agent - /// (`ceo`) under a company's workspace root. - fn skill_scratch(ws: &std::path::Path, slug: &str) -> std::path::PathBuf { - ws.join("acme") - .join("ceo") - .join("skill-catalog") - .join("skills") - .join(slug) - .join("SKILL.md") - } - - /// The regression: a skill authored in the console after the first roster - /// build reaches the agent on the NEXT `ensure` — the fingerprint changes, - /// the roster rebuilds in place, and the skill's `SKILL.md` materializes — - /// even though MCP / overlay / capability / composio are all unchanged. - #[tokio::test] - async fn ensure_rebuilds_when_a_custom_skill_is_authored() { - let skills = Arc::new(MemSkills::default()); - let mut fx = fixture(); - fx.deps.skills = Some(skills.clone()); - let ws = fx._dir.path().to_path_buf(); - let pool = HarnessPool::new(); - let rec = record(); - - pool.ensure(&rec, &fx.deps).await.expect("first ensure"); - let before = pool - .skill_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert!( - !skill_scratch(&ws, "standup-digest").exists(), - "no skill authored yet" - ); - - // Author a custom skill in the "console" (the live store) — no restart. - skills - .set(&rec.id, &custom_skill("standup-digest", true, STANDUP_MD)) - .await - .unwrap(); - - pool.ensure(&rec, &fx.deps).await.expect("second ensure"); - let after = pool - .skill_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert_ne!( - before, after, - "authoring a skill must change the fingerprint" - ); - assert_eq!( - pool.resident_companies().await, - 1, - "same company, rebuilt in place" - ); - assert!( - skill_scratch(&ws, "standup-digest").is_file(), - "the authored skill must surface to the agent with no restart" - ); - - // A third ensure with no change is a no-op (fingerprint stable). - pool.ensure(&rec, &fx.deps).await.expect("third ensure"); - assert_eq!(pool.skill_fingerprint_of(&rec.id).await, Some(after)); - } - - /// An unchanged delta set across two `ensure` calls keeps the fingerprint - /// stable and reuses the cached roster (the common fast path). - #[tokio::test] - async fn ensure_skill_fast_path_is_stable() { - let skills = Arc::new(MemSkills::default()); - let rec = record(); - skills - .set(&rec.id, &custom_skill("standup-digest", true, STANDUP_MD)) - .await - .unwrap(); - let mut fx = fixture(); - fx.deps.skills = Some(skills.clone()); - let pool = HarnessPool::new(); - - pool.ensure(&rec, &fx.deps).await.expect("first ensure"); - let first = pool.skill_fingerprint_of(&rec.id).await.unwrap(); - pool.ensure(&rec, &fx.deps).await.expect("second ensure"); - let second = pool.skill_fingerprint_of(&rec.id).await.unwrap(); - assert_eq!( - first, second, - "unchanged deltas keep the fingerprint stable" - ); - assert_eq!( - pool.resident_companies().await, - 1, - "roster reused, not grown" - ); - } - - /// Disabling a skill in the console drops it from the rebuilt scratch tree - /// on the next `ensure` (fingerprint moves, `SKILL.md` gone). - #[tokio::test] - async fn ensure_rebuilds_when_a_skill_is_disabled() { - let skills = Arc::new(MemSkills::default()); - let rec = record(); - skills - .set(&rec.id, &custom_skill("standup-digest", true, STANDUP_MD)) - .await - .unwrap(); - let mut fx = fixture(); - fx.deps.skills = Some(skills.clone()); - let ws = fx._dir.path().to_path_buf(); - let pool = HarnessPool::new(); - - pool.ensure(&rec, &fx.deps).await.expect("first ensure"); - let enabled_fp = pool.skill_fingerprint_of(&rec.id).await.unwrap(); - let path = skill_scratch(&ws, "standup-digest"); - assert!(path.is_file(), "an enabled skill materializes"); - - // Disable it in the console. - skills - .set(&rec.id, &custom_skill("standup-digest", false, STANDUP_MD)) - .await - .unwrap(); - pool.ensure(&rec, &fx.deps).await.expect("second ensure"); - let disabled_fp = pool.skill_fingerprint_of(&rec.id).await.unwrap(); - assert_ne!(enabled_fp, disabled_fp, "disabling changes the fingerprint"); - assert!( - !path.exists(), - "a disabled skill is dropped from the rebuilt scratch tree" - ); - assert_eq!(pool.resident_companies().await, 1, "rebuilt in place"); - } - - /// The fingerprint is order-agnostic (the store gives no ordering contract) - /// but content-sensitive (an edited `custom_doc` must trigger a rebuild). - #[test] - fn skill_delta_fingerprint_is_order_agnostic_but_content_sensitive() { - let a = custom_skill("alpha", true, "---\nname: A\ndescription: a\n---\n"); - let b = custom_skill("beta", true, "---\nname: B\ndescription: b\n---\n"); - assert_eq!( - skill_delta_fingerprint(&[a.clone(), b.clone()]), - skill_delta_fingerprint(&[b, a.clone()]), - "row order must not change the fingerprint" - ); - - let a_edited = custom_skill("alpha", true, "---\nname: A\ndescription: EDITED\n---\n"); - assert_ne!( - skill_delta_fingerprint(&[a]), - skill_delta_fingerprint(&[a_edited]), - "an edited custom_doc must change the fingerprint" - ); - } - - // --- Overlay-agent freshness (issue #71) -------------------------------- - - /// A `CompanyStore` backed by a live, mutable record — so a test can mutate - /// it between two `ensure` calls the same way the console `POST .../team` - /// route or the orchestrator's `add_agent` tool would, and observe the - /// freshness gate react. - #[derive(Default)] - struct LiveStore { - record: StdMutex>, - } - - #[async_trait] - impl CompanyStore for LiveStore { - async fn load(&self, _id: &CompanyId) -> crate::Result> { - Ok(self.record.lock().unwrap().clone()) - } - async fn save(&self, record: &CompanyRecord) -> crate::Result<()> { - *self.record.lock().unwrap() = Some(record.clone()); - Ok(()) - } - async fn list(&self) -> crate::Result> { - Ok(Vec::new()) - } - async fn append_ledger(&self, _id: &CompanyId, _entry: LedgerEntry) -> crate::Result<()> { - Ok(()) - } - } - - /// An overlay teammate added through the live company store (the same path - /// the console `POST .../team` route and the orchestrator's `add_agent` tool - /// both write through) reaches the roster on the company's NEXT `ensure` — - /// no restart — mirroring `ensure_rebuilds_when_a_runtime_mcp_server_is_added`. - #[tokio::test] - async fn ensure_rebuilds_when_an_overlay_agent_is_added() { - let live_store = Arc::new(LiveStore::default()); - let rec = record(); - live_store.save(&rec).await.unwrap(); - - let dir = tempfile::tempdir().unwrap(); - let deps = HarnessDeps { - ledgers: None, - ledger_registry: Default::default(), - provider: Arc::new(MockProvider::new("mock: ")), - provider_slug: "mock".to_string(), - context: Arc::new(MockContext::default()), - store: live_store.clone(), - meter: None, - workspace_root: dir.path().to_path_buf(), - workspace_git_enabled: false, - audit_root: dir.path().to_path_buf(), - model_override: None, - tasks: None, - artifacts: None, - skills: None, - skills_source_dir: None, - skills_registry: std::sync::Arc::from([]), - default_mcp_servers: Vec::new(), - mcp_servers: Vec::new(), - facts: None, - events: None, - delegations: DelegationQueue::default(), - workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), - mcp_failures: McpFailureQueue::default(), - pending_publishes: crate::harness::publish::PendingPublishQueue::default(), - workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), - run_outputs: crate::harness::orchestrator::RunOutputCache::default(), - run_output_store: None, - workflow_revisions: None, - approval_requests: ApprovalRequestQueue::default(), - secrets: None, - web_allowed_domains: Vec::new(), - capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, - workflow_source_dir: None, - plan: None, - media: None, - composio: None, - #[cfg(feature = "chargebee")] - chargebee: None, - #[cfg(feature = "paypal")] - paypal: None, - hosting: None, - steer: crate::company::steer::InflightRegistry::default(), - run_supervisor: crate::runtime::RunSupervisor::default(), - delivery: None, - search: None, - workspace: None, - repos: None, - repo_bindings: Vec::new(), - checkouts: crate::harness::repo::CheckoutLedger::default(), - }; - let pool = HarnessPool::new(); - - pool.ensure(&rec, &deps).await.expect("first ensure"); - let before = pool - .overlay_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert_eq!(pool.resident_companies().await, 1); - // The roster is not addressable under "growth" yet. - assert!( - pool.run(&rec.id, "growth", "hi", &deps, None) - .await - .is_err(), - "the overlay teammate must not exist before it is added" - ); - - // Add a teammate directly through the live store — the same write path - // `AddAgentTool` and the console `POST .../team` route both use. - let mut updated = rec.clone(); - updated.overlay_agents.push(OverlayAgent { - id: "growth".into(), - name: "Jamie".into(), - role: "Growth Lead".into(), - description: None, - tools: Vec::new(), - }); - live_store.save(&updated).await.unwrap(); - - // Next ensure re-resolves the live store → fingerprint changes → roster - // rebuilt, so the new teammate reaches the company without a restart. - pool.ensure(&rec, &deps).await.expect("second ensure"); - let after = pool - .overlay_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert_ne!( - before, after, - "adding a teammate must change the overlay fingerprint" - ); - assert_eq!( - pool.resident_companies().await, - 1, - "same company, rebuilt in place" - ); - - let reply = pool - .run(&rec.id, "growth", "hello-marker", &deps, None) - .await - .expect("the new teammate is addressable on the very next turn") - .reply; - assert!(reply.contains("hello-marker"), "got {reply:?}"); - - // A third ensure with no further change is a no-op (fingerprint stable). - pool.ensure(&rec, &deps).await.expect("third ensure"); - assert_eq!(pool.overlay_fingerprint_of(&rec.id).await, Some(after)); - } - - // --- Capability-budget freshness (issue #108) --------------------------- - - /// A manifest that grants every tool namespace, so the roster actually builds - /// the exec tools the capability filter then trims. (The default `manifest()` - /// grants nothing, so no exec tools would be present to gate.) - fn granting_manifest() -> CompanyManifest { - toml::from_str( - r#" -[company] -name = "Acme" - -[policy] -mode = "full" - -[tools] -allow = ["shell", "code", "web", "files"] - -[[agent]] -id = "ceo" -role = "Chief Executive" -description = "Sets direction." -"#, - ) - .expect("valid manifest") - } - - fn granting_record() -> CompanyRecord { - CompanyRecord { - id: CompanyId::new("acme"), - manifest: granting_manifest(), - ledger: Vec::new(), - lifecycle: "running".to_string(), - overlay_agents: Vec::new(), - overlay_desk_members: Vec::new(), - overlay_desk_order: Vec::new(), - overlay_desks: Vec::new(), - overlay_workflows: Vec::new(), - overlay_budgets: Vec::new(), - overlay_policy: None, - overlay_desk_tools: Default::default(), - disabled_workflows: Vec::new(), - template_provenance: None, - setup: None, - } - } - - /// The `ceo` roster agent's live tool names (test introspection via the - /// public `Agent::tools()` accessor). - async fn ceo_tool_names(pool: &HarnessPool, id: &CompanyId) -> Vec { - let guard = pool.agents.read().await; - let roster = guard.get(id).expect("roster present"); - let ceo = roster - .iter() - .find(|a| a.agent_id == "ceo") - .expect("ceo present"); - let agent = ceo.agent.lock().await; - agent.tools().iter().map(|t| t.name().to_string()).collect() - } - - /// End-to-end capability gating: a plan budgeting `shell` at 100 tokens grants - /// the shell tools while spend is under budget; once a recorded turn pushes - /// period spend past the threshold, the very next `ensure` rebuilds the roster - /// with the shell namespace dropped — while intrinsic tools (memory) and the - /// ungated `files` namespace survive. Mirrors the MCP-freshness test shape. - #[tokio::test] - async fn ensure_gates_shell_tools_once_the_token_budget_is_crossed() { - let dir = tempfile::tempdir().unwrap(); - let meter = Arc::new(RecordingMeter::default()); - let plan = crate::harness::capability_budget::CapabilityPlan { - period: crate::harness::capability_budget::BudgetPeriod::Daily, - budgets: std::collections::BTreeMap::from([("shell".to_string(), 100u64)]), - total_budget: None, - }; - let deps = HarnessDeps { - ledgers: None, - ledger_registry: Default::default(), - provider: Arc::new(MockProvider::new("mock: ")), - provider_slug: "mock".to_string(), - context: Arc::new(MockContext::default()), - store: Arc::new(RecordingStore::default()), - meter: Some(meter.clone()), - workspace_root: dir.path().to_path_buf(), - workspace_git_enabled: false, - audit_root: dir.path().to_path_buf(), - model_override: None, - tasks: None, - artifacts: None, - skills: None, - skills_source_dir: None, - skills_registry: std::sync::Arc::from([]), - default_mcp_servers: Vec::new(), - mcp_servers: Vec::new(), - facts: None, - events: None, - delegations: DelegationQueue::default(), - workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), - mcp_failures: McpFailureQueue::default(), - pending_publishes: crate::harness::publish::PendingPublishQueue::default(), - workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), - run_outputs: crate::harness::orchestrator::RunOutputCache::default(), - run_output_store: None, - workflow_revisions: None, - approval_requests: ApprovalRequestQueue::default(), - secrets: None, - web_allowed_domains: Vec::new(), - capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, - workflow_source_dir: None, - plan: Some(plan), - media: None, - composio: None, - #[cfg(feature = "chargebee")] - chargebee: None, - #[cfg(feature = "paypal")] - paypal: None, - hosting: None, - steer: crate::company::steer::InflightRegistry::default(), - run_supervisor: crate::runtime::RunSupervisor::default(), - delivery: None, - search: None, - workspace: None, - repos: None, - repo_bindings: Vec::new(), - checkouts: crate::harness::repo::CheckoutLedger::default(), - }; - let pool = HarnessPool::new(); - let rec = granting_record(); - - // First ensure: 0 spend < 100 → shell granted. - pool.ensure(&rec, &deps).await.expect("first ensure"); - let before_fp = pool - .capability_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - let before = ceo_tool_names(&pool, &rec.id).await; - assert!(before.contains(&"shell".to_string()), "got {before:?}"); - assert!( - before.contains(&"read_workspace_state".to_string()), - "got {before:?}" - ); - // `memory_store`/`memory_recall` are currently withheld altogether - // (see `harness::build::memory_tools`'s doc comment) — openhuman - // removed the constructor seam that let either tool act on a - // company's own `ContextStore` rather than one shared, - // unconfigured store. `file_read` is this test's example of an - // intrinsic, ungated tool instead. - assert!( - before.contains(&"file_read".to_string()), - "ungated files namespace must be present: {before:?}" - ); - - // Record a turn that burns 150 inference tokens — past the 100 budget. - meter - .record( - &rec.id, - &UsageSample { - at_millis: crate::ports::now_millis(), - agent: "ceo".into(), - provider: "managed".into(), - input_tokens: 100, - output_tokens: 50, - cached_input_tokens: 0, - cost_usd: 0.0, - kind: crate::ports::SampleKind::Inference, - run_id: None, - }, - ) - .await - .unwrap(); - - // Second ensure: 150 >= 100 → shell exhausted → roster rebuilt without it. - pool.ensure(&rec, &deps).await.expect("second ensure"); - let after_fp = pool - .capability_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert_ne!( - before_fp, after_fp, - "crossing the budget must change the capability fingerprint" - ); - assert_eq!(pool.resident_companies().await, 1, "rebuilt in place"); - - let after = ceo_tool_names(&pool, &rec.id).await; - assert!( - !after.contains(&"shell".to_string()), - "shell must be gated off once exhausted: {after:?}" - ); - assert!( - !after.contains(&"read_workspace_state".to_string()), - "the whole shell namespace drops: {after:?}" - ); - assert!( - after.contains(&"file_read".to_string()), - "ungated files namespace survives gating: {after:?}" - ); - - // Third ensure with no new spend → no rebuild (fingerprint stable). - pool.ensure(&rec, &deps).await.expect("third ensure"); - assert_eq!( - pool.capability_fingerprint_of(&rec.id).await, - Some(after_fp) - ); - } - - /// With no plan wired, the capability fingerprint is stable across ensures — - /// gating stays off, byte-identical to Cell A (no rebuild on this axis). - #[tokio::test] - async fn ensure_without_a_plan_never_gates() { - let fx = fixture(); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &fx.deps).await.expect("first ensure"); - let fp = pool - .capability_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - pool.ensure(&rec, &fx.deps).await.expect("second ensure"); - assert_eq!( - pool.capability_fingerprint_of(&rec.id).await, - Some(fp), - "no plan → stable fingerprint → no capability-driven rebuild" - ); - } - - /// Builds a `HarnessDeps` carrying the given plan + meter, for the total- - /// ceiling dispatch tests (issue #188). Everything else is the inert fixture - /// wiring (mock provider/context, recording store). - fn deps_with_plan( - dir: &std::path::Path, - context: Arc, - meter: Option>, - plan: Option, - ) -> HarnessDeps { - HarnessDeps { - ledgers: None, - ledger_registry: Default::default(), - provider: Arc::new(MockProvider::new("mock: ")), - provider_slug: "mock".to_string(), - context, - store: Arc::new(RecordingStore::default()), - meter, - workspace_root: dir.to_path_buf(), - workspace_git_enabled: false, - audit_root: dir.to_path_buf(), - model_override: None, - tasks: None, - skills: None, - skills_source_dir: None, - skills_registry: std::sync::Arc::from([]), - default_mcp_servers: Vec::new(), - mcp_servers: Vec::new(), - facts: None, - events: None, - delegations: DelegationQueue::default(), - workflow_runner: crate::harness::orchestrator::WorkflowRunnerHandle::default(), - mcp_failures: McpFailureQueue::default(), - pending_publishes: crate::harness::publish::PendingPublishQueue::default(), - workflow_refs: crate::harness::workflow_refs::WorkflowRefQueue::default(), - run_outputs: crate::harness::orchestrator::RunOutputCache::default(), - run_output_store: None, - workflow_revisions: None, - approval_requests: ApprovalRequestQueue::default(), - secrets: None, - web_allowed_domains: Vec::new(), - capabilities: crate::harness::toolbelt::CapabilityFilter::AllowAll, - workflow_source_dir: None, - plan, - media: None, - composio: None, - #[cfg(feature = "chargebee")] - chargebee: None, - #[cfg(feature = "paypal")] - paypal: None, - hosting: None, - artifacts: None, - steer: crate::company::steer::InflightRegistry::default(), - run_supervisor: crate::runtime::RunSupervisor::default(), - delivery: None, - search: None, - workspace: None, - repos: None, - repo_bindings: Vec::new(), - checkouts: crate::harness::repo::CheckoutLedger::default(), - } - } - - /// The hard total-token ceiling (issue #188): once the tenant's total period - /// spend crosses the plan's `total_budget`, the very next dispatch is refused - /// **before any model call** — the reply is the fixed operator notice, the - /// prompt is never echoed (proving the model was not run), and no fabricated - /// outcome lands in memory. A turn under the ceiling still runs normally. - #[tokio::test] - async fn run_refuses_dispatch_once_the_total_ceiling_is_crossed() { - let dir = tempfile::tempdir().unwrap(); - let context = Arc::new(MockContext::default()); - let meter = Arc::new(RecordingMeter::default()); - let plan = crate::harness::capability_budget::CapabilityPlan { - period: crate::harness::capability_budget::BudgetPeriod::Daily, - budgets: std::collections::BTreeMap::new(), - total_budget: Some(100), - }; - let deps = deps_with_plan( - dir.path(), - context.clone(), - Some(meter.clone() as Arc), - Some(plan), - ); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &deps).await.expect("ensure"); - - // Under the ceiling (0 spend < 100): the turn runs and echoes the prompt. - let ok = pool - .run(&rec.id, "ceo", "hello-marker", &deps, None) - .await - .expect("under-ceiling turn runs") - .reply; - assert!( - ok.contains("hello-marker"), - "under the ceiling the model runs: {ok:?}" - ); - - // Push total period spend to 150 — past the 100-token ceiling. - meter - .record( - &rec.id, - &UsageSample { - at_millis: crate::ports::now_millis(), - agent: "ceo".into(), - provider: "managed".into(), - input_tokens: 100, - output_tokens: 50, - cached_input_tokens: 0, - cost_usd: 0.0, - kind: crate::ports::SampleKind::Inference, - run_id: None, - }, - ) - .await - .unwrap(); - - let before = context - .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) - .await - .unwrap() - .len(); - - // Over the ceiling: dispatch is refused with a benign notice — NOT an Err. - let refused = pool - .run(&rec.id, "ceo", "should-not-echo", &deps, None) - .await - .expect("a refusal is a benign outcome, not a hard error") - .reply; - assert_eq!( - refused, TOTAL_BUDGET_EXHAUSTED_NOTICE, - "the refusal returns the fixed operator notice" - ); - assert!( - !refused.contains("should-not-echo"), - "the model was never called, so the prompt is not echoed: {refused:?}" - ); - - // A refused turn writes no outcome back to memory. - let after = context - .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) - .await - .unwrap() - .len(); - assert_eq!(before, after, "a refused turn stores nothing in memory"); - } - - /// Issue #416, the reason [`HarnessPool::total_ceiling_refusal`] was - /// extracted rather than copied: a confined turn reaches nothing, but it - /// still spends model tokens, so the tenant's ceiling refuses it exactly as - /// it refuses a roster dispatch. Without this test the gate could be dropped - /// from `run_confined` and every other test would stay green — the copilot - /// would simply keep spending past the cap. - #[tokio::test] - async fn a_confined_turn_is_refused_once_the_total_ceiling_is_crossed() { - let dir = tempfile::tempdir().unwrap(); - let context = Arc::new(MockContext::default()); - let meter = Arc::new(RecordingMeter::default()); - let plan = crate::harness::capability_budget::CapabilityPlan { - period: crate::harness::capability_budget::BudgetPeriod::Daily, - budgets: std::collections::BTreeMap::new(), - total_budget: Some(100), - }; - let deps = deps_with_plan( - dir.path(), - context.clone(), - Some(meter.clone() as Arc), - Some(plan), - ); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &deps).await.expect("ensure"); - - let confinement = confine::Confinement::workflow("weekly_report"); - let thread = Some("workflow-copilot:weekly_report"); - - // Under the ceiling the copilot answers, so the refusal below is the - // ceiling talking and not the confined path failing to run at all. - let ok = pool - .run_confined(&rec.id, "Acme", "hello-marker", &deps, thread, &confinement) - .await - .expect("under-ceiling confined turn runs") - .reply; - assert!( - ok.contains("hello-marker"), - "under the ceiling the model runs: {ok:?}" - ); - - // Push total period spend past the 100-token ceiling. - meter - .record( - &rec.id, - &UsageSample { - at_millis: crate::ports::now_millis(), - agent: "ceo".into(), - provider: "managed".into(), - input_tokens: 100, - output_tokens: 50, - cached_input_tokens: 0, - cost_usd: 0.0, - kind: crate::ports::SampleKind::Inference, - run_id: None, - }, - ) - .await - .unwrap(); - - let refused = pool - .run_confined( - &rec.id, - "Acme", - "should-not-echo", - &deps, - thread, - &confinement, - ) - .await - .expect("a refusal is a benign outcome, not a hard error") - .reply; - assert_eq!( - refused, TOTAL_BUDGET_EXHAUSTED_NOTICE, - "the copilot must not keep spending past the tenant ceiling" - ); - assert!( - !refused.contains("should-not-echo"), - "the model was never called, so the prompt is not echoed: {refused:?}" - ); - } - - /// Fail-closed tradeoff (issue #188): with a total ceiling configured but no - /// meter to read spend from, the hard refusal does NOT fire — a transient - /// unreadable-spend condition must not brick every turn. The turn runs (the - /// per-namespace fail-closed roster already handles exec-tool stripping). - #[tokio::test] - async fn run_does_not_refuse_when_spend_is_unreadable() { - let dir = tempfile::tempdir().unwrap(); - let context = Arc::new(MockContext::default()); - // A zero ceiling would refuse from the first token IF spend were readable; - // with no meter wired the gate must defer, not brick. - let plan = crate::harness::capability_budget::CapabilityPlan { - period: crate::harness::capability_budget::BudgetPeriod::Daily, - budgets: std::collections::BTreeMap::new(), - total_budget: Some(0), - }; - let deps = deps_with_plan(dir.path(), context.clone(), None, Some(plan)); - let pool = HarnessPool::new(); - let rec = record(); - pool.ensure(&rec, &deps).await.expect("ensure"); - - let reply = pool - .run(&rec.id, "ceo", "hello-marker", &deps, None) - .await - .expect("no meter must not brick the turn") - .reply; - assert!( - reply.contains("hello-marker"), - "an unreadable ceiling defers to running the turn: {reply:?}" - ); - assert_ne!( - reply, TOTAL_BUDGET_EXHAUSTED_NOTICE, - "the hard refusal must not fire without a spend read" - ); - } - - // --- The per-agent daily spend cap at dispatch (issue #304) -------------- - - /// A company whose `ceo` carries a $5/day cap and whose `engineer` carries - /// none — the pair that proves the gate is per-teammate, not per-company. - fn capped_record() -> CompanyRecord { - let manifest: CompanyManifest = toml::from_str( - r#" -[company] -name = "Acme" - -[policy] -mode = "full" - -[[agent]] -id = "ceo" -role = "Chief Executive" -description = "Sets direction." -budget_usd_daily = 5.0 - -[[agent]] -id = "engineer" -role = "Engineer" -description = "Builds the product." -"#, - ) - .expect("valid manifest"); - CompanyRecord { - manifest, - ..record() - } - } - - /// A `$usd` inference sample for `agent`, stamped at `at_millis`. - fn spend_sample(agent: &str, usd: f64, at_millis: u64) -> UsageSample { - UsageSample { - at_millis, - agent: agent.into(), - provider: "managed".into(), - input_tokens: 0, - output_tokens: 0, - cached_input_tokens: 0, - cost_usd: usd, - kind: crate::ports::SampleKind::Inference, - run_id: None, - } - } - - /// The heart of #304 at the layer that carries the money: once a teammate - /// has spent its manifest `budget_usd_daily`, its next dispatch is refused - /// **before any model call** — while its uncapped colleague keeps working. - /// - /// This is the layer that matters, because the dominant spend stream is - /// inference and inference never reaches a `ToolPolicy`. Gating only priced - /// tool calls would leave a capped teammate free to burn its budget many - /// times over on model turns alone. - #[tokio::test] - async fn run_refuses_dispatch_for_a_teammate_over_its_daily_cap() { - let dir = tempfile::tempdir().unwrap(); - let context = Arc::new(MockContext::default()); - let meter = Arc::new(RecordingMeter::default()); - let rec = capped_record(); - - // The CEO has spent its whole $5 today. The engineer has spent nothing. - meter - .record( - &rec.id, - &spend_sample("ceo", 5.00, crate::ports::now_millis()), - ) - .await - .unwrap(); - - let deps = deps_with_plan( - dir.path(), - context.clone(), - Some(meter.clone() as Arc), - None, - ); - let pool = HarnessPool::new(); - pool.ensure(&rec, &deps).await.expect("ensure"); - - let samples_before = meter.samples.lock().unwrap().len(); - let memory_before = context - .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) - .await - .unwrap() - .len(); - - let refused = pool - .run(&rec.id, "ceo", "should-not-echo", &deps, None) - .await - .expect("a refusal is a benign outcome, not a hard error") - .reply; - assert_eq!( - refused, - agent_budget_exhausted_notice("ceo", 5.0), - "the refusal names the teammate, its cap and the reset" - ); - assert!( - !refused.contains("should-not-echo"), - "the model was never called, so the prompt is not echoed: {refused:?}" - ); - assert_eq!( - meter.samples.lock().unwrap().len(), - samples_before, - "a pre-model-call refusal meters nothing" - ); - assert_eq!( - context - .list(&rec.id, memory_loop::OUTCOME_LABEL_PREFIX) - .await - .unwrap() - .len(), - memory_before, - "a refused turn stores no fabricated outcome" - ); - - // The cap is per-teammate: the uncapped engineer is untouched, and the - // CEO's spend does not count against it. - let ok = pool - .run(&rec.id, "engineer", "hello-marker", &deps, None) - .await - .expect("an uncapped teammate keeps working") - .reply; - assert!( - ok.contains("hello-marker"), - "one teammate's exhausted budget must not stop the company: {ok:?}" - ); - } - - // --- Console budget overrides, live (issue #343) ------------------------- - - /// **The no-restart proof.** A daily cap written through the company store — - /// the exact path `PUT …/team/{id}/budget` writes through — is enforced on - /// the company's **next dispatch**, in one process, with no restart and no - /// redeploy. - /// - /// This is the whole of #343 at the layer that decides whether a teammate - /// works. Before it, `budget_usd_daily` was readable only from the manifest, - /// which is a boot snapshot baked into the tenant image — so an operator - /// whose teammate had stopped had no remedy short of us shipping a new - /// image. The four phases walk exactly that operator's day: - /// - /// A. the CEO has spent its manifest $5 and is refused (issue #304, and - /// the state that motivates the issue); - /// B. an admin **raises** the cap to $50 — the stopped teammate works - /// again on its very next turn. This is the acceptance criterion; - /// C. the admin sets the cap to **$0** — a real cap of nothing, refused - /// from the first cent; - /// D. the admin **clears** the cap — an explicitly-uncapped override that - /// beats the manifest's $5 even with $5 already spent, so the teammate - /// works again. - /// - /// C and D are the same route with different bodies and they must not - /// resolve alike: C refuses, D runs. That is "clearing is distinct from - /// zeroing" asserted on live behaviour rather than on a type. - /// - /// Throughout, the pool holds **one** resident company and is never - /// reconstructed — `resident_companies()` stays 1 and the same `pool` binding - /// serves every phase — so the only mechanism that can be carrying these - /// changes is the budget fingerprint flipping and `ensure` rebuilding the - /// roster in place. Each phase asserts that fingerprint actually moved. - #[tokio::test] - async fn a_budget_written_through_the_store_is_enforced_on_the_next_dispatch() { - use crate::ports::types::{Actor, ActorKind, BudgetOverride}; - - let dir = tempfile::tempdir().unwrap(); - let context = Arc::new(MockContext::default()); - let meter = Arc::new(RecordingMeter::default()); - let rec = capped_record(); - - // A live store, so `ensure` re-resolves the overrides the way it does in - // production. `deps_with_plan`'s default store is inert. - let live_store = Arc::new(LiveStore::default()); - live_store.save(&rec).await.unwrap(); - - // The CEO has already spent its manifest $5 today. - meter - .record( - &rec.id, - &spend_sample("ceo", 5.00, crate::ports::now_millis()), - ) - .await - .unwrap(); - - let mut deps = deps_with_plan( - dir.path(), - context.clone(), - Some(meter.clone() as Arc), - None, - ); - deps.store = live_store.clone(); - - // ONE pool for the whole test. Nothing below reconstructs it, so nothing - // below can be smuggling in a restart. - let pool = HarnessPool::new(); - - /// Writes an override through the store exactly as the console route - /// does, and returns the record for the next `ensure`. - fn with_override(base: &CompanyRecord, cap: Option) -> CompanyRecord { - let mut next = base.clone(); - next.overlay_budgets = vec![BudgetOverride { - agent_id: "ceo".to_string(), - budget_usd_daily: cap, - set_by: Actor { - kind: ActorKind::User, - id: "user-admin".to_string(), - }, - at_millis: crate::ports::now_millis(), - }]; - next - } - - // --- A. The manifest cap is spent: the teammate is stopped. ---------- - pool.ensure(&rec, &deps).await.expect("ensure A"); - let fp_manifest = pool - .budget_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - let refused = pool - .run(&rec.id, "ceo", "should-not-echo", &deps, None) - .await - .expect("a refusal is a benign outcome") - .reply; - assert_eq!( - refused, - agent_budget_exhausted_notice("ceo", 5.0), - "phase A: the manifest's $5 cap is spent, so dispatch is refused" - ); - - // --- B. An admin raises the cap. The teammate works again. ----------- - live_store - .save(&with_override(&rec, Some(50.0))) - .await - .unwrap(); - pool.ensure(&rec, &deps).await.expect("ensure B"); - let fp_raised = pool - .budget_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert_ne!( - fp_manifest, fp_raised, - "phase B: setting a cap must move the budget fingerprint, or the \ - cached roster is reused and the change never reaches the gate" - ); - assert_eq!( - pool.resident_companies().await, - 1, - "phase B: the same company, rebuilt in place — not a new process" - ); - let unblocked = pool - .run(&rec.id, "ceo", "hello-marker", &deps, None) - .await - .expect("the raised cap unblocks the teammate") - .reply; - assert!( - unblocked.contains("hello-marker"), - "phase B: raising the cap from the console must unblock the stopped \ - teammate on its very next dispatch, with no restart: {unblocked:?}" - ); - - // --- C. The admin sets the cap to zero. Zero is a real cap. ---------- - live_store - .save(&with_override(&rec, Some(0.0))) - .await - .unwrap(); - pool.ensure(&rec, &deps).await.expect("ensure C"); - let fp_zero = pool - .budget_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert_ne!(fp_raised, fp_zero, "phase C: lowering a cap is a change"); - let zeroed = pool - .run(&rec.id, "ceo", "should-not-echo", &deps, None) - .await - .expect("a refusal is a benign outcome") - .reply; - assert_eq!( - zeroed, - agent_budget_exhausted_notice("ceo", 0.0), - "phase C: a $0 cap refuses from the first cent" - ); - - // --- D. The admin clears the cap. Cleared is not zero. --------------- - live_store.save(&with_override(&rec, None)).await.unwrap(); - pool.ensure(&rec, &deps).await.expect("ensure D"); - let fp_cleared = pool - .budget_fingerprint_of(&rec.id) - .await - .expect("fingerprinted"); - assert_ne!( - fp_zero, fp_cleared, - "phase D: 'no cap' and 'a cap of $0' must not hash alike — if they \ - did, clearing a cap would silently leave the teammate at zero" - ); - let cleared = pool - .run(&rec.id, "ceo", "hello-marker", &deps, None) - .await - .expect("an explicitly-uncapped teammate runs") - .reply; - assert!( - cleared.contains("hello-marker"), - "phase D: an explicitly-uncapped override beats the manifest's $5 \ - even with $5 already spent today: {cleared:?}" - ); - - // Nothing above restarted anything. - assert_eq!( - pool.resident_companies().await, - 1, - "one company, rebuilt in place across all four phases" - ); - - // A further `ensure` with no change is a no-op: the axis is not thrashing - // the roster (and dropping live agent sessions) on every turn. - pool.ensure(&rec, &deps).await.expect("ensure idempotent"); - assert_eq!(pool.budget_fingerprint_of(&rec.id).await, Some(fp_cleared)); - } - - /// An **overlay** teammate — one added from the console, with no manifest - /// row — can be capped through the same override, and is refused when it has - /// spent it. Before #343 an overlay teammate was unconditionally uncapped - /// ("overlay teammates are uncapped in v1"), so this is a capability that did - /// not exist rather than a behaviour that changed. - #[tokio::test] - async fn an_overlay_teammate_can_be_capped_from_the_console() { - use crate::ports::types::{Actor, ActorKind, BudgetOverride}; - - let dir = tempfile::tempdir().unwrap(); - let context = Arc::new(MockContext::default()); - let meter = Arc::new(RecordingMeter::default()); - - let mut rec = record(); - rec.overlay_agents.push(OverlayAgent { - id: "growth".into(), - name: "Jamie".into(), - role: "Growth Lead".into(), - description: None, - tools: Vec::new(), - }); - let live_store = Arc::new(LiveStore::default()); - live_store.save(&rec).await.unwrap(); - - let mut deps = deps_with_plan( - dir.path(), - context.clone(), - Some(meter.clone() as Arc), - None, - ); - deps.store = live_store.clone(); - let pool = HarnessPool::new(); - - // Uncapped to begin with: it answers. - pool.ensure(&rec, &deps).await.expect("ensure"); - let reply = pool - .run(&rec.id, "growth", "hello-marker", &deps, None) - .await - .expect("an uncapped overlay teammate answers") - .reply; - assert!(reply.contains("hello-marker"), "got {reply:?}"); - - // The operator caps it at $1 and it has already spent $2. - meter - .record( - &rec.id, - &spend_sample("growth", 2.00, crate::ports::now_millis()), - ) - .await - .unwrap(); - let mut capped = rec.clone(); - capped.overlay_budgets = vec![BudgetOverride { - agent_id: "growth".to_string(), - budget_usd_daily: Some(1.0), - set_by: Actor { - kind: ActorKind::User, - id: "user-admin".to_string(), - }, - at_millis: crate::ports::now_millis(), - }]; - live_store.save(&capped).await.unwrap(); - - pool.ensure(&rec, &deps).await.expect("ensure again"); - let refused = pool - .run(&rec.id, "growth", "should-not-echo", &deps, None) - .await - .expect("a refusal is a benign outcome") - .reply; - assert_eq!( - refused, - agent_budget_exhausted_notice("growth", 1.0), - "a console-added teammate is capped by the same gate as a manifest one" - ); - } - - /// Fail-open pin, mirroring #188's documented tradeoff exactly: with a cap - /// set but spend unreadable, the turn RUNS. - /// - /// A `$0` cap would refuse from the first cent if spend were readable, so a - /// meter that errors is the only reason this turn can proceed. Bricking a - /// teammate's cognition on a flaky read is a strictly worse failure mode - /// than one day of overspend — and unlike the policy arm's park, a turn-level - /// refusal offers the operator nothing to approve. - #[tokio::test] - async fn run_does_not_refuse_a_capped_teammate_when_spend_is_unreadable() { - let dir = tempfile::tempdir().unwrap(); - let context = Arc::new(MockContext::default()); - let manifest: CompanyManifest = toml::from_str( - r#" -[company] -name = "Acme" - -[policy] -mode = "full" - -[[agent]] -id = "ceo" -role = "Chief Executive" -description = "Sets direction." -budget_usd_daily = 0.0 -"#, - ) - .expect("valid manifest"); - let rec = CompanyRecord { - manifest, - ..record() - }; - - let deps = deps_with_plan( - dir.path(), - context.clone(), - Some(Arc::new(FailingMeter) as Arc), - None, - ); - let pool = HarnessPool::new(); - pool.ensure(&rec, &deps).await.expect("ensure"); - - let reply = pool - .run(&rec.id, "ceo", "hello-marker", &deps, None) - .await - .expect("an unreadable budget must not brick the teammate") - .reply; - assert!( - reply.contains("hello-marker"), - "an unreadable cap defers to running the turn: {reply:?}" - ); - - // ...and with no meter at all, the same deferral. - let no_meter = deps_with_plan(dir.path(), context.clone(), None, None); - let pool = HarnessPool::new(); - pool.ensure(&rec, &no_meter).await.expect("ensure"); - let reply = pool - .run(&rec.id, "ceo", "hello-marker", &no_meter, None) - .await - .expect("no meter must not brick the teammate") - .reply; - assert!(reply.contains("hello-marker"), "no meter defers: {reply:?}"); - } - - /// The cap is the UTC calendar day: yesterday's $9 does not refuse today's - /// first turn. Depends on `RecordingMeter` honouring `since_millis`. - #[tokio::test] - async fn a_yesterday_stamped_spend_does_not_refuse_todays_dispatch() { - let dir = tempfile::tempdir().unwrap(); - let context = Arc::new(MockContext::default()); - let meter = Arc::new(RecordingMeter::default()); - let rec = capped_record(); - - let yesterday = - crate::metering::utc_day_start_millis(crate::ports::now_millis()).saturating_sub(1); - meter - .record(&rec.id, &spend_sample("ceo", 9.00, yesterday)) - .await - .unwrap(); - - let deps = deps_with_plan( - dir.path(), - context.clone(), - Some(meter.clone() as Arc), - None, - ); - let pool = HarnessPool::new(); - pool.ensure(&rec, &deps).await.expect("ensure"); - - let reply = pool - .run(&rec.id, "ceo", "hello-marker", &deps, None) - .await - .expect("a new day admits the turn") - .reply; - assert!( - reply.contains("hello-marker"), - "the cap resets at 00:00Z; yesterday's spend is spent: {reply:?}" - ); - } - - // ----------------------------------------------------------------------- - // The approval gate's coverage over the live toolbelt (issue #443) - // ----------------------------------------------------------------------- - - /// Build one agent and return the tools it actually received. - /// - /// A local mirror of `build`'s own `built_tool_names` — that one is private - /// to its test module, and this file owns `deps_with_plan`, which is the - /// expensive half. - fn belt(grants: &[&str], is_orchestrator: bool, wire_everything: bool) -> Vec { - let dir = tempfile::tempdir().expect("tempdir"); - let mut deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); - if wire_everything { - // The three tool families gated on a wired dependency rather than - // on a cargo feature. Without these the belt is missing exactly the - // tools most likely to be misclassified — the workspace writes and - // the priced search. - deps.workspace = Some(Arc::new(crate::store::FsOps::new(dir.path()))); - deps.artifacts = Some(Arc::new(crate::store::FsOps::new(dir.path()))); - deps.search = Some(crate::harness::search::SearchBackend::new( - "https://api.example.test".to_string(), - crate::company::credentials::Credential::from_value("managed-platform-token"), - crate::company::DEFAULT_SEARCH_DAILY_CALLS, - )); - // Issue #245: a repository manager AND a binding, because the tools - // are gated on both — with a manager and nothing bound the belt - // would be missing `repo_checkout` / `repo_pr` and this check would - // pass while never having looked at them, which is the exact way - // `describe_skill` stayed invisible here while parking in - // production. - // Issue #752 added a fourth gate: a backend that keeps the - // credential off this container's disk. Declared here for the same - // reason the binding below is — without it the belt would be - // missing `repo_checkout` / `repo_pr` and this check would pass - // while never having looked at them. - deps.repos = Some(Arc::new( - crate::runtime::RepoManager::new( - CompanyId::new("acme"), - dir.path().join("repos"), - Arc::new(crate::store::FsSecretStore::new(dir.path())), - ) - .with_storage_kind(crate::store::StorageKind::Mongodb), - )); - deps.repo_bindings = vec![crate::runtime::repo_manager::types::RepoBinding { - key: "acme-widgets-000000000000".to_string(), - url: "https://github.com/acme/widgets".to_string(), - owner: "acme".to_string(), - repo: "widgets".to_string(), - branches: vec!["main".to_string()], - token_fingerprint: "0f1e2d3c4b5a".to_string(), - last_fetched_millis: None, - size_bytes: 0, - bound_at_millis: 1, - can_push: None, - }]; - // A registered MCP server is what puts `mcp_list_servers`, - // `mcp_list_tools` and `mcp_call_tool` on the belt — the three - // tools issue #443 is about. Without one the coverage check would - // pass while never having looked at them. - // A skills source dir is what puts `list_skills`, `describe_skill` - // and `read_skill_resource` on the belt (named for skills since - // issue #845; upstream calls them `*_workflow*`). Leaving it `None` - // is how those three stayed invisible to this check while - // `describe_workflow` parked in production. - let company_src = dir.path().join("company-src"); - std::fs::create_dir_all(company_src.join("skills").join("brief")).expect("skill dir"); - std::fs::write( - company_src.join("skills").join("brief").join("SKILL.md"), - "---\nname: brief\ndescription: Write a brief\n---\n\nWrite one.\n", - ) - .expect("skill file"); - deps.skills_source_dir = Some(company_src); - deps.mcp_servers = vec![McpServerDecl { - name: "notes".to_string(), - endpoint: "https://mcp.example.test".to_string(), - description: None, - allowed_tools: Vec::new(), - disallowed_tools: Vec::new(), - timeout_secs: 30, - enabled: true, - source: crate::company::mcp::McpSource::Runtime, - auth: crate::company::mcp::AuthMaterial::None, - }]; - } - let manifest_agent = ManifestAgent { - global: false, - id: "desk".to_string(), - role: "Desk Lead".to_string(), - name: None, - description: None, - tier: None, - tools: Vec::new(), - delegates_to: Vec::new(), - context: None, - budget_usd_daily: None, - prompt: None, - prompt_files: Vec::new(), - prompt_files_resolved: Vec::new(), - classes: Vec::new(), - ledgers: None, - can_declare_ledgers: true, - }; - let policy = ApprovalPolicy::new(&Policy::default(), None); - let grants: Vec = grants.iter().map(|g| g.to_string()).collect(); - let agent = build::build_agent( - &CompanyId::new("acme"), - "Acme", - &manifest_agent, - policy, - &deps, - &grants, - &[], - &[], - is_orchestrator, - ) - .expect("agent builds"); - agent.tools().iter().map(|t| t.name().to_string()).collect() - } - - /// **The mechanism issue #443 asks for.** Every tool this crate can put in - /// front of an agent must be classified in - /// [`crate::policy::consequence`], or this fails. - /// - /// Three families had needed the same carve-out before it, each added after - /// somebody hit it, and what the gate did with the ones nobody hit was - /// silent: the tool simply started asking for permission, and whoever - /// noticed was an operator wondering why a read needed approving. That is - /// how `mcp_list_servers` — which the agent persona *instructs* every agent - /// to call — came to cost an approval, and how `file_read`, `glob` and - /// `grep` came to park with nobody reporting it. - /// - /// A tool declaring its own consequence to the gate at call time would be - /// better, and is not reachable: openhuman's `ToolPolicy` surface hands the - /// bridge a name and arguments, never the tool. So the declaration is - /// checked against the live belt here instead — the issue's own stated - /// fallback, "exhaustive by construction rather than by memory". - /// - /// Feature-aware by construction: it enumerates whatever this build wires, - /// so a family behind a cargo feature is covered by the lane that enables - /// it rather than by a `cfg` branch that has to be kept in step. - #[test] - fn every_registered_tool_is_declared() { - let declared: std::collections::BTreeSet<&str> = - crate::policy::consequence::declared_tools().collect(); - let mut live: std::collections::BTreeSet = std::collections::BTreeSet::new(); - for (grants, orchestrator, everything) in [ - (&["*"][..], false, false), - (&["*"][..], true, false), - (&["*"][..], false, true), - (&["*"][..], true, true), - ( - &["workspace", "search", "media", "composio", "repo"][..], - false, - true, - ), - ] { - live.extend(belt(grants, orchestrator, everything)); - } - // A vacuity guard with teeth. `!live.is_empty()` would not notice the - // belt quietly narrowing to the tools nobody was worried about, and - // three of the four names below are the ones the issues are about. - for expected in [ - "shell", - "workspace_write", - "file_read", - "describe_skill", - "repo_checkout", - #[cfg(feature = "mcp")] - "mcp_list_servers", - #[cfg(feature = "mcp")] - "mcp_call_tool", - ] { - assert!( - live.contains(expected), - "the belt builder stopped wiring `{expected}`, so this check has \ - narrowed without anyone deciding to narrow it: {live:?}" - ); - } - let undeclared: Vec<&String> = live - .iter() - .filter(|name| !declared.contains(name.as_str())) - .collect(); - assert!( - undeclared.is_empty(), - "these tools are wired onto a live agent but nobody has said what they can \ - reach, so the gate is guessing from their names and they cannot be granted \ - standing: {undeclared:?}. Add them to `crate::policy::consequence::DECLARED`." - ); - } - - /// The one-directional cross-check on the declaration. - /// - /// A tool's own `permission_level()` is NOT trustworthy as the authority — - /// it defaults to `ReadOnly`, and upstream tools that plainly mutate - /// (`git_operations`, `memory_store`) never override it, so believing a - /// `ReadOnly` claim would wave a write straight through the gate. But the - /// claims in the *other* direction are deliberate: nothing declares itself - /// `Execute` or `Dangerous` by accident. So those are checked, and a - /// `ReadOnly` claim is ignored. - #[test] - fn nothing_that_declares_itself_executable_is_internal_or_grantable() { - use oh::tools::traits::PermissionLevel; - let dir = tempfile::tempdir().expect("tempdir"); - let deps = deps_with_plan(dir.path(), Arc::new(MockContext::default()), None, None); - let manifest_agent = ManifestAgent { - global: false, - id: "desk".to_string(), - role: "Desk Lead".to_string(), - name: None, - description: None, - tier: None, - tools: Vec::new(), - delegates_to: Vec::new(), - context: None, - budget_usd_daily: None, - prompt: None, - prompt_files: Vec::new(), - prompt_files_resolved: Vec::new(), - classes: Vec::new(), - ledgers: None, - can_declare_ledgers: true, - }; - let agent = build::build_agent( - &CompanyId::new("acme"), - "Acme", - &manifest_agent, - ApprovalPolicy::new(&Policy::default(), None), - &deps, - &["*".to_string()], - &[], - &[], - true, - ) - .expect("agent builds"); - let args = serde_json::json!({}); - let mut checked = 0; - for tool in agent.tools() { - if !matches!( - tool.permission_level(), - PermissionLevel::Execute | PermissionLevel::Dangerous - ) { - continue; - } - checked += 1; - let verdict = crate::policy::consequence_of(tool.name(), &args); - assert!( - verdict.reach.denied_under_readonly(), - "`{}` declares itself executable but a read-only desk would allow it", - tool.name() - ); - assert!( - !verdict.standing.is_grantable(), - "`{}` declares itself executable and must not be grantable", - tool.name() - ); - } - assert!(checked > 0, "no executable tool was on the belt to check"); - } - - // --- Per-company billing resolution (issues #788, #789) ----------------- - // - // `resolve_chargebee` / `resolve_paypal` are what actually decide whether a - // company's agents get billing tools on a given turn — `HarnessPool::ensure` - // re-resolves them every turn, and `RuntimeBuilder::build` runs the same - // three-way decision once at boot. All three branches are silent when they - // go wrong: a dropped grant check wires tools the manifest never allowed, and - // a read error collapsed into "no credential" disconnects a working - // integration on one transient store hiccup. - - /// A secret store that reads back what was seeded, or fails every read. - #[cfg(any(feature = "chargebee", feature = "paypal"))] - #[derive(Default)] - struct BillingSecrets { - map: StdMutex>, - fail: bool, - } - - #[cfg(any(feature = "chargebee", feature = "paypal"))] - #[async_trait] - impl SecretStore for BillingSecrets { - async fn get( - &self, - _c: &CompanyId, - key: &str, - ) -> crate::Result> { - if self.fail { - return Err(crate::error::OpenCompanyError::Store( - "the secret store is unreachable".into(), - )); - } - Ok(self - .map - .lock() - .unwrap() - .get(key) - .map(|v| crate::ports::types::SecretValue(v.clone()))) - } - async fn set( - &self, - _c: &CompanyId, - key: &str, - value: crate::ports::types::SecretValue, - ) -> crate::Result<()> { - self.map.lock().unwrap().insert(key.to_string(), value.0); - Ok(()) - } - } - - /// A company whose manifest allows exactly `grants`. - #[cfg(any(feature = "chargebee", feature = "paypal"))] - fn record_granting(grants: &[&str]) -> CompanyRecord { - let mut rec = record(); - rec.manifest.tools.allow = grants.iter().map(|g| g.to_string()).collect(); - rec - } - - /// The inert fixture deps, with a secret store and a "last known" connection. - #[cfg(any(feature = "chargebee", feature = "paypal"))] - fn billing_deps(dir: &std::path::Path, secrets: Arc) -> HarnessDeps { - let mut deps = deps_with_plan(dir, Arc::new(MockContext::default()), None, None); - deps.secrets = Some(secrets); - deps - } - - #[cfg(feature = "chargebee")] - #[tokio::test] - async fn chargebee_resolves_only_for_a_company_that_grants_it() { - let dir = tempfile::tempdir().expect("tempdir"); - let secrets = Arc::new(BillingSecrets::default()); - secrets - .set( - &CompanyId::new("acme"), - crate::chargebee::types::SITE_SECRET, - crate::ports::types::SecretValue("acme-test".into()), - ) - .await - .expect("seed"); - secrets - .set( - &CompanyId::new("acme"), - crate::chargebee::types::API_KEY_SECRET, - crate::ports::types::SecretValue("cb_key".into()), - ) - .await - .expect("seed"); - let deps = billing_deps(dir.path(), secrets); - let pool = HarnessPool::new(); - - // Granted and configured: the credential resolves. - let granted = pool - .resolve_chargebee(&record_granting(&["chargebee"]), &deps) - .await - .expect("a granted, configured company resolves"); - assert_eq!(granted.site(), "acme-test"); - - // Same credentials, no grant. The store is untouched — the gate is the - // manifest, so a company that never opted in gets no tools however well - // configured the host happens to be. - assert!( - pool.resolve_chargebee(&record_granting(&[]), &deps) - .await - .is_none(), - "an ungranted company must resolve nothing" - ); - - // And a wildcard is not a grant: these tools send invoices to real - // people, so they are opted into by name rather than riding in on the - // `*` somebody set for file and shell tools. - assert!( - pool.resolve_chargebee(&record_granting(&["*"]), &deps) - .await - .is_none(), - "a catch-all grant must not confer chargebee" - ); - } - - #[cfg(feature = "chargebee")] - #[tokio::test] - async fn a_chargebee_store_hiccup_keeps_the_last_known_connection() { - // The distinction this pins: absence wires no tools, but a READ FAILURE - // keeps whatever was already resolved. Collapsing the two would drop a - // working company's billing tools mid-conversation on one bad read, and - // silently — the agent would simply stop being able to invoice. - let dir = tempfile::tempdir().expect("tempdir"); - let secrets = Arc::new(BillingSecrets { - fail: true, - ..Default::default() - }); - let mut deps = billing_deps(dir.path(), secrets); - let last_known = crate::harness::chargebee::TenantChargebee::resolve( - &(Arc::new(BillingSecrets { - map: StdMutex::new( - [ - ( - crate::chargebee::types::SITE_SECRET.to_string(), - "acme-test".to_string(), - ), - ( - crate::chargebee::types::API_KEY_SECRET.to_string(), - "cb_key".to_string(), - ), - ] - .into_iter() - .collect(), - ), - fail: false, - }) as Arc), - &CompanyId::new("acme"), - ) - .await - .expect("the seeded store reads") - .expect("both halves present"); - deps.chargebee = Some(last_known); - - let kept = pool_resolve_chargebee(&deps).await; - assert_eq!( - kept.map(|c| c.site().to_string()).as_deref(), - Some("acme-test"), - "a transient read failure must not disconnect a working integration" - ); - } - - #[cfg(feature = "chargebee")] - async fn pool_resolve_chargebee( - deps: &HarnessDeps, - ) -> Option { - HarnessPool::new() - .resolve_chargebee(&record_granting(&["chargebee"]), deps) - .await - } - - #[cfg(feature = "paypal")] - #[tokio::test] - async fn paypal_resolves_only_for_a_company_that_grants_it() { - let dir = tempfile::tempdir().expect("tempdir"); - let secrets = Arc::new(BillingSecrets::default()); - for (key, value) in [ - (crate::company::paypal::CLIENT_ID_SECRET, "AY_id"), - (crate::company::paypal::CLIENT_SECRET_SECRET, "EL_secret"), - ] { - secrets - .set( - &CompanyId::new("acme"), - key, - crate::ports::types::SecretValue(value.into()), - ) - .await - .expect("seed"); - } - let deps = billing_deps(dir.path(), secrets); - let pool = HarnessPool::new(); - - assert!( - pool.resolve_paypal(&record_granting(&["paypal"]), &deps) - .await - .is_some(), - "a granted, configured company resolves" - ); - assert!( - pool.resolve_paypal(&record_granting(&[]), &deps) - .await - .is_none(), - "an ungranted company must resolve nothing" - ); - assert!( - pool.resolve_paypal(&record_granting(&["*"]), &deps) - .await - .is_none(), - "a catch-all grant must not confer paypal" - ); - } - - #[cfg(feature = "paypal")] - #[tokio::test] - async fn a_paypal_grant_with_no_credential_wires_nothing_rather_than_failing() { - // Fail closed: a manifest that grants `paypal` on a host where nobody - // has saved a credential must wire no tools, not tools that fail on - // first use — an agent that HAS a wallet tool tells the operator the - // balance is unavailable, rather than that it cannot read wallets. - let dir = tempfile::tempdir().expect("tempdir"); - let deps = billing_deps(dir.path(), Arc::new(BillingSecrets::default())); - assert!( - HarnessPool::new() - .resolve_paypal(&record_granting(&["paypal"]), &deps) - .await - .is_none() - ); - } -} +/// Aliased rather than glob-re-exported because [`built_in`] has its own +/// `run_turn` module — the two would collide under a bare re-export. +#[cfg(feature = "acp")] +pub use acp::run_turn as acp_run_turn; diff --git a/src/harness/router.rs b/src/harness/router.rs new file mode 100644 index 000000000..fbe238cf8 --- /dev/null +++ b/src/harness/router.rs @@ -0,0 +1,577 @@ +//! [`HarnessRouter`]: sending each agent's turn to the harness it is bound to. +//! +//! ## Why this is a router and not a setting +//! +//! Which engine runs a turn used to be one decision per company, taken at boot +//! from "did an inference credential resolve?". That made two things impossible +//! that a company actually wants: a roster spanning a cheap model and an +//! expensive one, and a single coding agent on the operator's own Claude Code +//! while everyone else stays on the embedded loop. +//! +//! [`RunTurn`] already carries `agent_id` on all three of its methods, so the +//! dispatch point was always there — nothing had ever varied on it. This type is +//! that seam: it holds one inner [`RunTurn`] per declared harness and forwards +//! each call to the one its agent names. +//! +//! ## Resolution, and why unbound agents are not an error +//! +//! An agent naming no harness runs on the company's default. That is not +//! leniency — it is what makes named harnesses additive: every roster written +//! before this existed binds nobody, and all of them must keep working. A +//! *named* harness that does not exist is a different matter and is rejected by +//! manifest validation long before a turn is attempted. +//! +//! ## What a missing engine means +//! +//! A harness can be declared, valid, and still have no engine here — an `acp` +//! harness in a build compiled without the `acp` feature, or a `built_in` one on +//! a host that resolved no inference. Those turns fail with a message naming the +//! harness and the reason, rather than silently falling back to another agent's +//! engine. Falling back would be the worst outcome available: the turn would +//! succeed, on a model and a credential nobody chose, and the only evidence +//! would be a billing line. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; + +use crate::Result; +use crate::company::steer::SteerControl; +use crate::error::OpenCompanyError; +use crate::harness::built_in::TurnOutcome; +use crate::harness::built_in::run_trace::RunTraceSink; +use crate::ports::types::{CompanyId, CompanyRecord}; +use crate::runtime::delegation::RunTurn; + +/// Routes each agent's turn to the [`RunTurn`] of the harness it is bound to. +pub struct HarnessRouter { + /// The harness id agents naming none run on. + default_id: String, + /// Agent id → harness id, for agents that named one. Agents absent from + /// this map take [`default_id`](Self::default_id). + by_agent: HashMap, + /// Harness id → the engine that serves it. A declared harness with no entry + /// here is one this build or host cannot run; see the module docs. + engines: HashMap>, + /// Why a declared harness has no engine, so the failure can say which + /// harness and what to do rather than "not found". + unavailable: HashMap, + /// Harness id → why its engine's last warm-up failed. A lane with a + /// recorded failure fails its turns with this reason while every other lane + /// keeps working; a successful re-`ensure` clears the entry, so a recovered + /// harness comes back without a restart. + failures: Mutex>, +} + +impl HarnessRouter { + /// A router over `default_id`, with no bindings and no engines yet. + pub fn new(default_id: impl Into) -> Self { + Self { + default_id: default_id.into(), + by_agent: HashMap::new(), + engines: HashMap::new(), + unavailable: HashMap::new(), + failures: Mutex::new(HashMap::new()), + } + } + + /// Registers the engine serving `harness_id`. + pub fn with_engine(mut self, harness_id: impl Into, engine: Arc) -> Self { + self.engines.insert(harness_id.into(), engine); + self + } + + /// Records that `harness_id` was declared but cannot run here, and why. + /// + /// `reason` is shown to the operator, so it should name the fix — "this + /// build has no `acp` feature", not "unsupported". + pub fn with_unavailable( + mut self, + harness_id: impl Into, + reason: impl Into, + ) -> Self { + self.unavailable.insert(harness_id.into(), reason.into()); + self + } + + /// Binds `agent_id` to `harness_id`. + pub fn bind(mut self, agent_id: impl Into, harness_id: impl Into) -> Self { + self.by_agent.insert(agent_id.into(), harness_id.into()); + self + } + + /// A router over `default_harness`, seeded with the default lane, every + /// extra lane, every unavailable harness, and every agent→harness binding. + /// + /// The single place both the brain and the workflow runner assemble a + /// router from the same four pieces, so the two dispatch points cannot + /// drift about which agent lands on which engine. + pub fn from_lanes( + default_harness: &str, + default_lane: Arc, + lanes: &[(String, Arc)], + unavailable: &[(String, String)], + bindings: &HashMap, + ) -> Self { + let mut router = Self::new(default_harness).with_engine(default_harness, default_lane); + for (id, engine) in lanes { + router = router.with_engine(id, engine.clone()); + } + for (id, reason) in unavailable { + router = router.with_unavailable(id, reason); + } + for (agent, harness) in bindings { + router = router.bind(agent, harness); + } + router + } + + /// The harness id `agent_id` runs on. + pub fn harness_for(&self, agent_id: &str) -> &str { + self.by_agent + .get(agent_id) + .map(String::as_str) + .unwrap_or(&self.default_id) + } + + /// The engine for `agent_id`, or the error explaining why there is none. + fn engine_for(&self, agent_id: &str) -> Result<&Arc> { + let harness = self.harness_for(agent_id); + if let Some(reason) = self.failures.lock().expect("router failures").get(harness) { + return Err(OpenCompanyError::Config(format!( + "agent `{agent_id}` is bound to harness `{harness}`, whose last warm-up failed: {reason}." + ))); + } + if let Some(engine) = self.engines.get(harness) { + return Ok(engine); + } + let detail = + self.unavailable.get(harness).map(String::as_str).unwrap_or( + "no engine was wired for it — this host cannot run turns on this harness", + ); + Err(OpenCompanyError::Config(format!( + "agent `{agent_id}` is bound to harness `{harness}`, but {detail}." + ))) + } +} + +#[async_trait] +impl RunTurn for HarnessRouter { + async fn run( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + chat_id: Option<&str>, + ) -> Result { + self.engine_for(agent_id)? + .run(company, agent_id, message, chat_id) + .await + } + + async fn run_steered( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + control: &SteerControl, + chat_id: Option<&str>, + run_sink: Option>, + ) -> Result { + self.engine_for(agent_id)? + .run_steered(company, agent_id, message, control, chat_id, run_sink) + .await + } + + async fn run_steered_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + control: &SteerControl, + run_sink: Option>, + ) -> Result { + self.engine_for(agent_id)? + .run_steered_background(company, agent_id, message, control, run_sink) + .await + } + + async fn run_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + ) -> Result { + self.engine_for(agent_id)? + .run_background(company, agent_id, message) + .await + } + + async fn ensure(&self, company: &CompanyRecord) -> Result<()> { + // Warm every engine's roster before the first turn, recording each + // lane's failure rather than stopping at the first: one bad lane must + // not take every other agent down with it. A declared harness with no + // engine here (an `acp` harness in a build without the feature, say) is + // not an engine to warm — its turn fails later with the reason, which is + // the point of `unavailable`. A lane that warms cleanly on a later + // `ensure` clears its recorded failure, so recovery needs no restart. + let mut outcomes = Vec::with_capacity(self.engines.len()); + for (harness, engine) in &self.engines { + outcomes.push((harness.clone(), engine.ensure(company).await)); + } + let mut failures = self.failures.lock().expect("router failures"); + for (harness, result) in outcomes { + match result { + Ok(()) => { + failures.remove(&harness); + } + Err(err) => { + failures.insert(harness, err.to_string()); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + + /// An engine that records which agent it was asked to run, so a test can + /// assert on *which* harness served a turn rather than only that one did. + #[derive(Default)] + struct SpyEngine { + label: String, + seen: Mutex>, + } + + impl SpyEngine { + fn new(label: &str) -> Arc { + Arc::new(Self { + label: label.to_string(), + seen: Mutex::new(Vec::new()), + }) + } + } + + #[async_trait] + impl RunTurn for SpyEngine { + async fn run( + &self, + _company: &CompanyId, + agent_id: &str, + _message: &str, + _chat_id: Option<&str>, + ) -> Result { + self.seen.lock().unwrap().push(agent_id.to_string()); + Ok(TurnOutcome { + reply: self.label.clone(), + steps: Vec::new(), + hit_iteration_cap: false, + halted_for_spend: None, + }) + } + + async fn run_steered( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + _control: &SteerControl, + chat_id: Option<&str>, + _run_sink: Option>, + ) -> Result { + self.run(company, agent_id, message, chat_id).await + } + + async fn run_steered_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + _control: &SteerControl, + _run_sink: Option>, + ) -> Result { + self.run(company, agent_id, message, None).await + } + } + + /// An engine whose `ensure` can be made to fail on command, so a test can + /// check that one lane's warm-up failure does not take every other lane + /// down, and that a later successful `ensure` brings the lane back. + struct FlakyEngine { + label: String, + fail_ensure: Mutex, + } + + impl FlakyEngine { + fn new(label: &str) -> Arc { + Arc::new(Self { + label: label.to_string(), + fail_ensure: Mutex::new(false), + }) + } + + fn set_fail(&self, fail: bool) { + *self.fail_ensure.lock().unwrap() = fail; + } + } + + #[async_trait] + impl RunTurn for FlakyEngine { + async fn run( + &self, + _company: &CompanyId, + _agent_id: &str, + _message: &str, + _chat_id: Option<&str>, + ) -> Result { + Ok(TurnOutcome { + reply: self.label.clone(), + steps: Vec::new(), + hit_iteration_cap: false, + halted_for_spend: None, + }) + } + + async fn run_steered( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + _control: &SteerControl, + chat_id: Option<&str>, + _run_sink: Option>, + ) -> Result { + self.run(company, agent_id, message, chat_id).await + } + + async fn run_steered_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + _control: &SteerControl, + _run_sink: Option>, + ) -> Result { + self.run(company, agent_id, message, None).await + } + + async fn ensure(&self, _company: &CompanyRecord) -> Result<()> { + if *self.fail_ensure.lock().unwrap() { + return Err(OpenCompanyError::Config( + "roster warm-up failed".to_string(), + )); + } + Ok(()) + } + } + + /// A minimal record for `ensure` to warm against; the engines in these + /// tests ignore it, so only the manifest-less shape is needed. + fn record() -> CompanyRecord { + let manifest: crate::company::CompanyManifest = + toml::from_str("[company]\nname = \"Acme\"\n").expect("manifest parses"); + CompanyRecord { + id: company(), + manifest, + ledger: Vec::new(), + lifecycle: "running".to_string(), + overlay_agents: Vec::new(), + overlay_desk_members: Vec::new(), + overlay_desk_order: Vec::new(), + overlay_desks: Vec::new(), + overlay_workflows: Vec::new(), + overlay_budgets: Vec::new(), + overlay_policy: None, + overlay_desk_tools: Default::default(), + disabled_workflows: Vec::new(), + template_provenance: None, + setup: None, + } + } + + fn company() -> CompanyId { + CompanyId::new("acme") + } + + /// The headline: two agents in one company, two engines, and each turn goes + /// to the one its agent named. + #[tokio::test] + async fn each_agent_runs_on_the_harness_it_names() { + let embedded = SpyEngine::new("embedded"); + let deep = SpyEngine::new("deep"); + let router = HarnessRouter::new("embedded") + .with_engine("embedded", embedded.clone()) + .with_engine("deep", deep.clone()) + .bind("researcher", "deep"); + + let out = router + .run(&company(), "researcher", "hi", None) + .await + .unwrap(); + assert_eq!(out.reply, "deep"); + + let out = router.run(&company(), "ceo", "hi", None).await.unwrap(); + assert_eq!(out.reply, "embedded", "an unbound agent takes the default"); + + assert_eq!(&*deep.seen.lock().unwrap(), &["researcher".to_string()]); + assert_eq!(&*embedded.seen.lock().unwrap(), &["ceo".to_string()]); + } + + /// Every `RunTurn` method routes, not just the streamed one. A method + /// that forwarded to a fixed engine would send *dispatched card* turns to + /// the wrong model while operator chat looked correct. + #[tokio::test] + async fn every_run_turn_method_routes() { + let embedded = SpyEngine::new("embedded"); + let deep = SpyEngine::new("deep"); + let router = HarnessRouter::new("embedded") + .with_engine("embedded", embedded.clone()) + .with_engine("deep", deep.clone()) + .bind("researcher", "deep"); + let control = SteerControl::default(); + + assert_eq!( + router + .run_steered(&company(), "researcher", "hi", &control, None, None) + .await + .unwrap() + .reply, + "deep" + ); + assert_eq!( + router + .run_steered_background(&company(), "researcher", "hi", &control, None) + .await + .unwrap() + .reply, + "deep" + ); + assert_eq!( + router + .run_background(&company(), "researcher", "hi") + .await + .unwrap() + .reply, + "deep" + ); + assert!( + embedded.seen.lock().unwrap().is_empty(), + "no method leaked to the default engine" + ); + } + + /// A harness with no engine fails the turn, naming the harness and the + /// reason. It must never quietly borrow another harness's engine: that turn + /// would succeed on a model and a credential nobody chose. + #[tokio::test] + async fn a_harness_with_no_engine_fails_rather_than_falling_back() { + let embedded = SpyEngine::new("embedded"); + let router = HarnessRouter::new("embedded") + .with_engine("embedded", embedded.clone()) + .with_unavailable( + "my_laptop", + "this build was compiled without the `acp` feature", + ) + .bind("coder", "my_laptop"); + + let err = router + .run(&company(), "coder", "hi", None) + .await + .expect_err("must not fall back"); + let msg = err.to_string(); + assert!(msg.contains("coder"), "{msg}"); + assert!(msg.contains("my_laptop"), "{msg}"); + assert!(msg.contains("`acp` feature"), "names the fix: {msg}"); + assert!( + embedded.seen.lock().unwrap().is_empty(), + "the default engine was never reached" + ); + } + + /// A binding to a harness nobody declared still fails closed, even though + /// manifest validation should have caught it first. Defence in depth: the + /// router is also reachable from runtime-constructed rosters that no + /// manifest validated. + #[tokio::test] + async fn an_unknown_harness_binding_fails_closed() { + let router = HarnessRouter::new("embedded").with_engine("embedded", SpyEngine::new("e")); + let err = router + .run(&company(), "ghost_bound", "hi", None) + .await + .expect("agent is unbound, so it takes the default") + .reply; + assert_eq!(err, "e"); + + let router = router.bind("ghost_bound", "nowhere"); + assert!( + router + .run(&company(), "ghost_bound", "hi", None) + .await + .is_err() + ); + } + + /// One lane failing to warm does not take down the others: `ensure` warms + /// every engine, records the failed lane, and only that lane's turns error. + #[tokio::test] + async fn one_lane_failing_to_warm_does_not_take_down_the_others() { + let embedded = FlakyEngine::new("embedded"); + let deep = FlakyEngine::new("deep"); + let router = HarnessRouter::new("embedded") + .with_engine("embedded", embedded.clone()) + .with_engine("deep", deep.clone()) + .bind("researcher", "deep") + .bind("ceo", "embedded"); + + deep.set_fail(true); + router.ensure(&record()).await.unwrap(); + + let err = router + .run(&company(), "researcher", "hi", None) + .await + .expect_err("the failed lane's turn must error"); + let msg = err.to_string(); + assert!(msg.contains("researcher"), "{msg}"); + assert!(msg.contains("deep"), "{msg}"); + assert!(msg.contains("warm-up"), "names the failed warm-up: {msg}"); + + let out = router.run(&company(), "ceo", "hi", None).await.unwrap(); + assert_eq!(out.reply, "embedded", "the healthy lane keeps working"); + } + + /// A lane that failed to warm comes back once a later `ensure` succeeds — + /// the recorded failure is cleared, so recovery needs no restart. + #[tokio::test] + async fn a_failed_lane_recovers_on_a_later_ensure() { + let embedded = FlakyEngine::new("embedded"); + let deep = FlakyEngine::new("deep"); + let router = HarnessRouter::new("embedded") + .with_engine("embedded", embedded.clone()) + .with_engine("deep", deep.clone()) + .bind("researcher", "deep"); + + deep.set_fail(true); + router.ensure(&record()).await.unwrap(); + assert!( + router + .run(&company(), "researcher", "hi", None) + .await + .is_err(), + "the failed lane errors before recovery" + ); + + deep.set_fail(false); + router.ensure(&record()).await.unwrap(); + let out = router + .run(&company(), "researcher", "hi", None) + .await + .unwrap(); + assert_eq!(out.reply, "deep", "recovery needs no restart"); + } +} diff --git a/src/harness/spend_halt_turn_test.rs b/src/harness/spend_halt_turn_test.rs index a81e36e63..70b9d60e3 100644 --- a/src/harness/spend_halt_turn_test.rs +++ b/src/harness/spend_halt_turn_test.rs @@ -18,7 +18,7 @@ //! real [`HostedProvider`], real pool, real brain — and scripts exactly one //! thing: the model's choices, over a loopback OpenAI-compatible endpoint. The //! shape [`cap_turn_test`](super::cap_turn_test) and -//! [`iteration_cap_turn_test`](super::iteration_cap_turn_test) established. +//! `built_in::iteration_cap_turn_test` established. //! //! The lever that makes the brake fire is `prompt_tokens`: the stop-hook //! middleware folds the usage the *provider* reports into openhuman's turn cost, @@ -309,6 +309,7 @@ fn deps_for(base_url: String, dir: &std::path::Path) -> (HarnessDeps, Arc extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(ops.clone()), diff --git a/src/metering/mod.rs b/src/metering/mod.rs index 4a9b9a390..aac4b6337 100644 --- a/src/metering/mod.rs +++ b/src/metering/mod.rs @@ -119,6 +119,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: vec![], delegates_to: vec![], context: None, @@ -137,6 +138,7 @@ mod tests { name: None, description: None, tier: None, + harness: None, tools: vec![], delegates_to: vec![], context: None, diff --git a/src/metering/triage.rs b/src/metering/triage.rs index 65a78ea9d..739f92bea 100644 --- a/src/metering/triage.rs +++ b/src/metering/triage.rs @@ -59,18 +59,21 @@ pub fn triage_sample(usage: &TokenUsage, provider: &str) -> Option } /// Records one completed triage escalation: the Finances ledger entry (when it -/// cost USD) and the usage sample (when it moved tokens or money). +/// cost USD) and, when a usage meter is wired, the usage sample. /// /// The ledger entry goes through the same [`inference_ledger_entry`] the cycle's /// inference spend uses, under the same `inference.spend` kind — triage spend is /// inference spend as far as the money is concerned, and only the *usage* -/// breakdown cares about the distinction. +/// breakdown cares about the distinction. The meter is deliberately optional: +/// a host with no usage meter still records the spend it can prove, exactly as +/// [`record_turn_cost`](crate::harness::cost::record_turn_cost) preserves its +/// ledger write without a meter. pub async fn record_triage_usage( usage: &TokenUsage, provider: &str, company: &CompanyId, store: &dyn CompanyStore, - meter: &dyn UsageMeter, + meter: Option<&dyn UsageMeter>, ) { if usage.is_zero() { return; @@ -94,6 +97,7 @@ pub async fn record_triage_usage( ); } if let Some(sample) = triage_sample(usage, provider) + && let Some(meter) = meter && let Err(err) = meter.record(company, &sample).await { tracing::warn!( @@ -106,7 +110,12 @@ pub async fn record_triage_usage( #[cfg(test)] mod test { + use std::sync::Mutex; + + use async_trait::async_trait; + use super::*; + use crate::ports::types::{CompanyRecord, CompanySummary, LedgerEntry}; fn usage() -> TokenUsage { TokenUsage { @@ -117,6 +126,109 @@ mod test { } } + #[derive(Default)] + struct RecordingStore { + ledger: Mutex>, + } + + #[async_trait] + impl CompanyStore for RecordingStore { + async fn load(&self, _id: &CompanyId) -> crate::Result> { + Ok(None) + } + async fn save(&self, _record: &CompanyRecord) -> crate::Result<()> { + Ok(()) + } + async fn list(&self) -> crate::Result> { + Ok(Vec::new()) + } + async fn append_ledger(&self, _id: &CompanyId, entry: LedgerEntry) -> crate::Result<()> { + self.ledger.lock().unwrap().push(entry); + Ok(()) + } + } + + #[derive(Default)] + struct RecordingMeter { + samples: Mutex>, + } + + #[async_trait] + impl UsageMeter for RecordingMeter { + async fn record(&self, _company: &CompanyId, sample: &UsageSample) -> crate::Result<()> { + self.samples.lock().unwrap().push(sample.clone()); + Ok(()) + } + async fn query( + &self, + _company: &CompanyId, + _since: u64, + ) -> crate::Result> { + Ok(self.samples.lock().unwrap().clone()) + } + } + + /// The meter is optional, the ledger is not: a host with no usage meter must + /// still record the spend it can prove (same contract as + /// [`record_turn_cost`](crate::harness::cost::record_turn_cost)). + #[tokio::test] + async fn meter_none_still_records_the_ledger_row() { + let store = RecordingStore::default(); + let company = CompanyId::new("acme"); + record_triage_usage(&usage(), "openrouter", &company, &store, None).await; + + let ledger = store.ledger.lock().unwrap(); + assert_eq!( + ledger.len(), + 1, + "the spend row must survive without a meter" + ); + let entry = &ledger[0]; + assert_eq!(entry.kind, super::super::inference::INFERENCE_SPEND_KIND); + assert_eq!(entry.memo, UNATTRIBUTED_AGENT); + assert!( + (entry.amount_usd - (-0.0004)).abs() < 1e-9, + "an outflow posts negative (issue #1047)" + ); + } + + /// A wired meter receives the same sample `triage_sample` builds — the + /// `record` call and the shape the aggregation reads are one contract. + #[tokio::test] + async fn a_wired_meter_records_the_sample_and_the_ledger() { + let store = RecordingStore::default(); + let meter = RecordingMeter::default(); + let company = CompanyId::new("acme"); + record_triage_usage(&usage(), "openrouter", &company, &store, Some(&meter)).await; + + let samples = meter.samples.lock().unwrap(); + assert_eq!(samples.len(), 1); + assert_eq!(samples[0].kind, SampleKind::TriageCall); + assert_eq!(samples[0].agent, UNATTRIBUTED_AGENT); + assert_eq!(samples[0].provider, "openrouter"); + assert_eq!(samples[0].input_tokens, 120); + assert_eq!(store.ledger.lock().unwrap().len(), 1); + } + + /// The offline path is a no-op at the record level too — nothing to charge + /// and nothing to meter. + #[tokio::test] + async fn a_zero_usage_escalation_records_nothing() { + let store = RecordingStore::default(); + let meter = RecordingMeter::default(); + let company = CompanyId::new("acme"); + record_triage_usage( + &TokenUsage::default(), + "managed", + &company, + &store, + Some(&meter), + ) + .await; + assert!(store.ledger.lock().unwrap().is_empty()); + assert!(meter.samples.lock().unwrap().is_empty()); + } + #[test] fn a_completed_escalation_is_charged_to_the_company_not_a_teammate() { let sample = triage_sample(&usage(), "managed").expect("a sample for real spend"); diff --git a/src/runtime/builder.rs b/src/runtime/builder.rs index 0b055a8bc..c7cc73f83 100644 --- a/src/runtime/builder.rs +++ b/src/runtime/builder.rs @@ -9,6 +9,8 @@ //! `build` performs boot replay: it loads the runtime journal and rehydrates //! any parked approvals into the gate so an approval survives a restart. +#[cfg(feature = "openhuman")] +use std::collections::HashMap; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; @@ -29,8 +31,12 @@ use crate::feedback::tinyhumans::TinyHumansClient; use crate::feedback::tool::BuiltinToolProvider; use crate::feedback::types::ConsentMode; #[cfg(feature = "openhuman")] +use crate::harness::built_in::run_turn::HarnessRunTurn; +#[cfg(feature = "openhuman")] use crate::harness::provider::{HostedProviderConfig, TenantProvider}; #[cfg(feature = "openhuman")] +use crate::harness::router::HarnessRouter; +#[cfg(feature = "openhuman")] use crate::harness::{HarnessBrain, HarnessDeps}; use crate::openhuman::rpc::OpenHumanRpc; use crate::openhuman::{OpenHumanChannelAdapter, OpenHumanToolProvider}; @@ -46,6 +52,8 @@ use crate::ports::{ SkillStateStore, TaskStore, ToolProvider, UsageMeter, UserStore, WorkflowRevisionStore, WorkspaceStore, }; +#[cfg(feature = "openhuman")] +use crate::runtime::delegation::RunTurn; // Separate line (#241) so this addition is a pure append, not a reflow of the // grouped import that sibling store-seam branches (#274, #596) also edit. use crate::ports::ScheduleFireStore; @@ -2163,9 +2171,21 @@ impl RuntimeBuilder { // managed env default? A corrupt runtime config degrades // to "unconfigured" (managed/echo brain) rather than // bricking boot. + // + // The manifest layer is the *default harness's* effective + // inference — `default_harness_inference()` falling back + // to the company-level `[inference]` — the same resolution + // `TenantProvider::new` applies a few lines down. A + // company whose only inference lives in + // `[harness.inference]` must count as configured here, + // or it would never reach the provider it just declared. + let effective_manifest = self + .manifest + .default_harness_inference() + .unwrap_or_else(|| self.manifest.inference.clone()); let configured = inference::resolve_effective( &id, - &self.manifest.inference, + &effective_manifest, env_default.as_ref(), secrets.as_ref(), ) @@ -2343,23 +2363,31 @@ impl RuntimeBuilder { Vec::new() }), ); - let deps = HarnessDeps { + let mut deps = HarnessDeps { // Carried so live re-resolution merges the same // three layers boot did (issue #527). default_mcp_servers: self.default_mcp_servers.clone(), // A per-tenant provider that re-resolves the // effective inference config on every turn, so a // console BYOK switch takes effect next turn with - // no rebuild. + // no rebuild. The default harness's own + // `[harness.inference]` beats the company-level + // `[inference]` — the same precedence a named + // harness gets — while the scope stays the + // default one so the flat legacy secret keys keep + // working for the company's default harness. provider: Arc::new(TenantProvider::new( id.clone(), secrets.clone(), - self.manifest.inference.clone(), - env_default, + self.manifest + .default_harness_inference() + .unwrap_or_else(|| self.manifest.inference.clone()), + env_default.clone(), )), // Static fallback only; `HarnessPool::run` reads // the live slug from the provider per turn. provider_slug: "managed".to_string(), + serves: None, context: context.clone(), store: store.clone(), meter: Some(fs_ops.clone()), @@ -2612,14 +2640,72 @@ impl RuntimeBuilder { template_provenance: template_provenance.clone(), setup: setup.clone(), }; - // Workflow agent nodes execute on the same pool as the - // brain — clone before both moves into `HarnessBrain`. - let runner: Arc = - Arc::new(HarnessWorkflowRunner::new( - pool.clone(), - deps.clone(), - record.clone(), - )); + // The company's other declared harnesses, each on + // its own pool and its own provider. Empty unless + // `[[harness]]` names more than one, so a company + // that declares none keeps exactly the single-pool + // path it always had. + // + // Built first so `deps.serves` is set before any + // dependency clones it — otherwise the runner (which + // holds `deps.clone()`) would carry `serves: None`, + // and `HarnessPool::ensure` (which does not fingerprint + // `serves`) could build the whole roster on the + // default provider regardless of which agents it + // actually serves. + let lanes = crate::harness::lanes::build( + &record, + &deps, + secrets.clone(), + env_default, + ); + if !lanes.lanes.is_empty() || !lanes.unavailable.is_empty() { + tracing::info!( + company = %id, + lanes = lanes.lanes.len(), + unavailable = lanes.unavailable.len(), + "wired named harnesses" + ); + } + // Narrow the default pool to the agents it actually + // serves once other lanes exist; `None` (the + // single-harness case) keeps the whole roster. + deps.serves = lanes.default_serves; + + // The router every dispatch goes through: the default + // lane plus each named lane, indexed by agent. Shared + // by the brain and the workflow runner so they cannot + // disagree about which agent lands on which engine. + let default_lane: Arc = + Arc::new(HarnessRunTurn::new(pool.clone(), Arc::new(deps.clone()))); + let turn: Arc = if lanes.lanes.is_empty() + && lanes.unavailable.is_empty() + { + default_lane + } else { + let default_harness = record.manifest.default_harness_id(); + let bindings: HashMap = record + .manifest + .agents + .iter() + .filter_map(|a| a.harness.clone().map(|h| (a.id.clone(), h))) + .collect(); + Arc::new(HarnessRouter::from_lanes( + &default_harness, + default_lane, + &lanes.lanes, + &lanes.unavailable, + &bindings, + )) + }; + + // Workflow agent nodes route through the shared + // router, so a workflow node addressing a named-lane + // agent lands on that lane's engine instead of the + // default pool. + let runner: Arc = Arc::new( + HarnessWorkflowRunner::new(turn, deps.clone(), record.clone()), + ); // Issue #67: fill the shared handle on `deps` (a clone // of which the runner holds, and which moves into the // brain below) so the orchestrator's `run_workflow` tool @@ -2653,7 +2739,10 @@ impl RuntimeBuilder { // choke point mints into and the boot reaper // sweeps, so an attempt's trace, cost and // status all land on the row it opened. - HarnessBrain::new(pool, deps, record).with_runs(ops.runs.clone()), + HarnessBrain::new(pool, deps, record) + .with_lanes(lanes.lanes) + .with_unavailable_lanes(lanes.unavailable) + .with_runs(ops.runs.clone()), ) as Arc) } else { // Do not degrade silently (issue #174): an openhuman diff --git a/src/runtime/delegation.rs b/src/runtime/delegation.rs index c99efbae1..4a82c81ba 100644 --- a/src/runtime/delegation.rs +++ b/src/runtime/delegation.rs @@ -93,6 +93,31 @@ pub trait RunTurn: Send + Sync { control: &SteerControl, run_sink: Option>, ) -> Result; + + /// An un-streamed, un-steered turn — a workflow agent node, which drops its + /// steps and shows no operator chat bubble. Its transient frames must not + /// reach the console timeline, which is the same reason this method exists + /// beside [`run_steered_background`](Self::run_steered_background) rather + /// than reusing [`run`](Self::run). + /// + /// Defaults to [`run`](Self::run) so the sentinel and test doubles need not + /// re-declare the same nothing; the streaming harness engines override it + /// to suppress the live stream. + async fn run_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + ) -> Result { + self.run(company, agent_id, message, None).await + } + + /// Warms whatever roster this engine caches before the first turn. The + /// default is a no-op; a harness that builds its roster lazily behind a + /// pool overrides it so a caller can ensure every lane before dispatch. + async fn ensure(&self, _company: &CompanyRecord) -> Result<()> { + Ok(()) + } } // `desk_lead` is the brain-agnostic desk-lead resolver — it moved to diff --git a/src/server/operator.rs b/src/server/operator.rs index 312e03a92..8436afb6e 100644 --- a/src/server/operator.rs +++ b/src/server/operator.rs @@ -3651,6 +3651,7 @@ mode = "full" run_supervisor: crate::runtime::RunSupervisor::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(home.to_path_buf())), store: Arc::new(FsCompanyStore::new(home.to_path_buf())), meter: Some(Arc::new(FsOps::new(home.to_path_buf()))), diff --git a/src/server/ops/inference.rs b/src/server/ops/inference.rs index a2cd02860..c10bf594d 100644 --- a/src/server/ops/inference.rs +++ b/src/server/ops/inference.rs @@ -147,10 +147,26 @@ struct SetInference { key: Option, } -/// Loads the company's committed `[inference]` section from its record. +/// Loads the inference the company actually boots and runs on: the *default +/// harness's* `[harness.inference]` when that harness declares one, falling back +/// to the company-level `[inference]` section. +/// +/// This mirrors [`RuntimeBuilder::build`](crate::runtime::RuntimeBuilder::build), +/// which resolves `default_harness_inference()` before the company-level +/// fallback. The status, probe, and runner-gap paths all read this, so a company +/// whose only inference lives in `[harness.inference]` must resolve here too — +/// otherwise it would report `managed`, reject `/inference/test` as +/// `not_configured`, and mislabel its status after a reset, while turns run on +/// the harness configuration the same record holds. async fn manifest_inference(runtime: &CompanyRuntime) -> Result { let record = runtime.store().load(runtime.id()).await.map_err(ApiError)?; - Ok(record.map(|r| r.manifest.inference).unwrap_or_default()) + Ok(record + .map(|r| { + r.manifest + .default_harness_inference() + .unwrap_or_else(|| r.manifest.inference.clone()) + }) + .unwrap_or_default()) } /// The console-facing source label for a resolved source badge. @@ -345,12 +361,12 @@ async fn effective_status_with( Some(platform) => resolve_effective(runtime.id(), &manifest, Some(platform), secrets) .await .map_err(ApiError)? - .map_or_else(|| inference::MANAGED_BASE_URL.to_string(), |d| d.base_url), + .map_or_else(|| inference::PLATFORM_BASE_URL.to_string(), |d| d.base_url), // No platform endpoint on this deployment: nothing to inherit, so the // tenant resolve already holds the whole answer and the second read is // skipped. None => decl.as_ref().map_or_else( - || inference::MANAGED_BASE_URL.to_string(), + || inference::PLATFORM_BASE_URL.to_string(), |d| d.base_url.clone(), ), }; @@ -490,7 +506,17 @@ async fn set_config( /// set: reverting decides which model the company thinks with. async fn revert_config(company: AdminScopedCompany) -> Result, ApiError> { let runtime = company.runtime.as_ref(); - clear_runtime_config(runtime.id(), runtime.secrets().as_ref()) + let secrets = runtime.secrets(); + clear_runtime_config(runtime.id(), secrets.as_ref()) + .await + .map_err(ApiError)?; + // Also clear any stored credential: a "Reset to managed" is supposed to be a + // full reset, not a half-clear that leaves a stale credential behind. The + // stored key would otherwise make `keyConfigured` appear false in the UI while + // secretly still being present, and the console's remove-key button would + // remain hidden — leaving the operator stranded with a credential they cannot + // clear. See issue #993 / inference.spec.ts cleanup. + inference::clear_key(runtime.id(), secrets.as_ref()) .await .map_err(ApiError)?; Ok(Json(MutationResponse { @@ -656,6 +682,32 @@ mod tests { toml::from_str("[company]\nname = \"Acme\"\n[policy]\nmode = \"full\"\n").unwrap() } + /// A manifest whose **only** inference lives in the default harness's + /// `[harness.inference]` — no company-level `[inference]` section at all. + /// `openhuman`-gated like its only caller: under the default build the + /// harness-wiring path the fix targets is compiled out, and an unused + /// helper would trip `clippy -D warnings`. + #[cfg(feature = "openhuman")] + fn manifest_with_harness_inference() -> CompanyManifest { + toml::from_str( + r#"[company] +name = "Acme" +[policy] +mode = "full" + +[[harness]] +id = "embedded" +kind = "built_in" +default = true + +[harness.inference] +provider = "openai_compatible" +base_url = "https://byo.example/v1" +"#, + ) + .unwrap() + } + /// Commits `manifest` as `id`'s record — what `manifest_inference` reads. async fn save_record(home: &std::path::Path, id: &CompanyId, manifest: &CompanyManifest) { use crate::ports::CompanyStore; @@ -695,6 +747,26 @@ mod tests { state } + /// [`state_with_company`] over a harness-only-inference manifest. The + /// routes read `manifest_inference` from the saved record, so the company + /// boots on the echo brain here (no pool attached) while the record it + /// reads still carries the harness's `[harness.inference]` — exactly the + /// shape of company the fix targets. + #[cfg(feature = "openhuman")] + async fn state_with_harness_inference(home: &std::path::Path) -> AppState { + let id = CompanyId::new("acme"); + save_record(home, &id, &manifest_with_harness_inference()).await; + let runtime = RuntimeBuilder::new(home.to_path_buf(), manifest_with_harness_inference()) + .with_id(id.clone()) + .build() + .await + .unwrap(); + let state = AppState::new(AppConfig::default()); + state.registry().insert(id, std::sync::Arc::new(runtime)); + crate::server::test_support::seed_fixed_admin(&state, "acme").await; + state + } + /// A rebuilder that rebuilds over the handover, as the binary's does. struct Working { home: std::path::PathBuf, @@ -865,7 +937,7 @@ mod tests { // No platform endpoint on this deployment — the built-in constant is // still the only honest answer, and this arm must not regress. let dto = effective_status_with(&runtime, None).await.unwrap(); - assert_eq!(dto.base_url, inference::MANAGED_BASE_URL); + assert_eq!(dto.base_url, inference::PLATFORM_BASE_URL); // Pointed at staging, the card follows — and *only* the URL moves. let dto = effective_status_with(&runtime, Some(&staging_platform())) @@ -898,7 +970,7 @@ mod tests { let runtime = runtime_with(home_dir.path(), MANAGED_MANIFEST).await; let dto = effective_status_with(&runtime, None).await.unwrap(); - assert_eq!(dto.base_url, inference::MANAGED_BASE_URL); + assert_eq!(dto.base_url, inference::PLATFORM_BASE_URL); let dto = effective_status_with(&runtime, Some(&staging_platform())) .await @@ -948,12 +1020,13 @@ mod tests { .unwrap(); assert!( dto.key_configured, - "a console-set key must read as configured even on managed" + "a console-set key must read as configured" ); // Same company, same injected platform default, opposite answer — and the - // endpoint is unmoved either way: paying for your own agents on the - // managed brain does not take you off it. - assert_eq!(dto.base_url, STAGING_URL); + // key is not merely recorded: it moves the company off the subscription + // proxy and onto its own OpenRouter account, which is the only way a + // stored `sk-or-…` could actually be used. + assert_eq!(dto.base_url, inference::OPENROUTER_BASE_URL); } #[tokio::test] @@ -972,25 +1045,60 @@ mod tests { assert_eq!(dto.base_url, "https://byo.example/v1"); } + /// A third-party endpoint we hold no credential for uses its own URL + /// verbatim — the platform default is not a fallback for somewhere we cannot + /// authenticate anyway. #[tokio::test] - async fn a_non_managed_provider_ignores_the_platform_default() { - // Only `managed` inherits the platform endpoint; every other kind uses - // its own resolved URL verbatim. + async fn a_third_party_provider_ignores_the_platform_default() { let home_dir = home(); let runtime = runtime_with( home_dir.path(), "[company]\nname = \"Acme\"\n[policy]\nmode = \"full\"\n\ - [inference]\nprovider = \"openrouter\"\n", + [inference]\nprovider = \"ollama\"\nbase_url = \"http://localhost:11434/v1\"\n", ) .await; let dto = effective_status_with(&runtime, Some(&staging_platform())) .await .unwrap(); - assert_eq!(dto.base_url, inference::OPENROUTER_BASE_URL); + assert_eq!(dto.base_url, "http://localhost:11434/v1"); assert_eq!(dto.source, "manifest"); } + /// `openrouter` is dual-mode, and which mode it is in depends only on + /// whether the tenant holds a key. With none it rides the subscription on + /// the platform endpoint; that is the config a company starts on, and it + /// must work with nothing configured. + #[tokio::test] + async fn keyless_openrouter_rides_the_subscription_and_a_key_goes_direct() { + let home_dir = home(); + let runtime = runtime_with( + home_dir.path(), + "[company]\nname = \"Acme\"\n[policy]\nmode = \"full\"\n\ + [inference]\nprovider = \"openrouter\"\n", + ) + .await; + let platform = staging_platform(); + + let dto = effective_status_with(&runtime, Some(&platform)) + .await + .unwrap(); + assert_eq!(dto.base_url, STAGING_URL, "proxied"); + assert_eq!(dto.slug, "subscription"); + assert!(!dto.key_configured); + + inference::store_key(runtime.id(), runtime.secrets().as_ref(), "sk-or-tenant") + .await + .unwrap(); + + let dto = effective_status_with(&runtime, Some(&platform)) + .await + .unwrap(); + assert_eq!(dto.base_url, inference::OPENROUTER_BASE_URL, "direct"); + assert_eq!(dto.slug, "openrouter"); + assert!(dto.key_configured); + } + /// The probe's gate stays keyed on *tenant* config. Pointing a deployment at /// a platform endpoint gives the probe somewhere real to aim, but it must not /// turn "nothing configured" into a live probe of the platform brain — that @@ -1007,6 +1115,37 @@ mod tests { assert_eq!(body["code"], "not_configured"); } + /// A company whose only inference lives in `[harness.inference]` resolves + /// the harness's provider for both status and probe — the same + /// default-harness fallback [`RuntimeBuilder::build`] applies at boot. + /// + /// Before the fix `manifest_inference` read only the company-level + /// `[inference]`, so such a company reported `managed`, rejected + /// `/inference/test` as `not_configured`, and mislabeled its status while + /// turns ran on the harness configuration the same record holds. + #[cfg(feature = "openhuman")] + #[tokio::test] + async fn harness_only_inference_reports_the_harness_provider_for_status_and_probe() { + let home_dir = home(); + let state = state_with_harness_inference(home_dir.path()).await; + + // Status resolves the default harness's `[harness.inference]`, not the + // absent company-level section: the operator sees the provider their + // turns actually run on. + let (status, dto, _) = send(&state, "GET", "/api/v1/company/inference", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(dto["provider"], "openai_compatible"); + assert_eq!(dto["source"], "manifest"); + assert_ne!(dto["provider"], "managed"); + + // The probe resolves the same inference, so it is *not* rejected as + // `not_configured`; it reaches the (unreachable) host and reports that + // failure instead. + let (status, body, _) = send(&state, "POST", "/api/v1/company/inference/test", None).await; + assert_ne!(status, StatusCode::CONFLICT); + assert_ne!(body["code"], "not_configured"); + } + #[cfg(feature = "openhuman")] #[test] fn platform_default_follows_the_injected_inference_url() { @@ -1092,13 +1231,66 @@ mod tests { assert_eq!(status, StatusCode::OK); assert_eq!(resp["status"]["provider"], "managed"); assert_eq!(resp["status"]["source"], "managed"); + assert_eq!( + resp["status"]["keyConfigured"], false, + "the reset must clear a stored credential too, or keyConfigured lies" + ); + } + + /// The reset is a *full* reset (issue #993): reverting also clears a stored + /// key. This is what keeps a keyless reconfiguration keyless — without it, a + /// stale secret would make the company resolve direct even though the console + /// shows no key, and `DELETE` would strand it with a credential it can never + /// see or clear. + #[tokio::test] + async fn revert_clears_the_key_so_a_keyless_save_rides_the_subscription() { + let home_dir = home(); + let home = home_dir.path().to_path_buf(); + let state = state_with_company(&home).await; + + // Store a key first, so the reset has something stale to clear. + let (status, resp, _) = send( + &state, + "PUT", + "/api/v1/company/inference", + Some(json!({ "provider": "openrouter", "key": TOKEN })), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(resp["status"]["slug"], "openrouter"); + assert_eq!(resp["status"]["keyConfigured"], true); + + let (status, resp, _) = send(&state, "DELETE", "/api/v1/company/inference", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(resp["status"]["keyConfigured"], false); + + // A keyless save afterwards must land on the subscription, not be flung + // direct by a credential the reset was supposed to remove. + let (status, resp, raw) = send( + &state, + "PUT", + "/api/v1/company/inference", + Some(json!({ "provider": "openrouter" })), + ) + .await; + assert_eq!(status, StatusCode::OK, "{raw}"); + assert_eq!(resp["status"]["slug"], "subscription"); + assert_eq!(resp["status"]["keyConfigured"], false); + + let (_, dto, raw) = send(&state, "GET", "/api/v1/company/inference", None).await; + assert_eq!(dto["slug"], "subscription"); + assert_eq!(dto["keyConfigured"], false); + assert!(!raw.contains(TOKEN), "GET leaked the reset token: {raw}"); } - /// Issue #585: the company's key on the *managed* brain — set, rotate, and - /// clear — is the whole point of the screen, and the route must never echo - /// any of the three tokens back. + /// Issue #585: the company's own key — set, rotate, and clear — is the whole + /// point of the screen, and the route must never echo any of the three + /// tokens back. + /// + /// Written against the legacy `managed` provider name on purpose: it is what + /// a console built before the rename still sends, and it must keep working. #[tokio::test] - async fn managed_key_can_be_set_rotated_and_cleared() { + async fn a_legacy_managed_key_can_be_set_rotated_and_cleared() { const ROTATED: &str = "sk-rotated-inference-token-ABC"; let home_dir = home(); let home = home_dir.path().to_path_buf(); @@ -1113,14 +1305,15 @@ mod tests { ) .await; assert_eq!(status, StatusCode::OK, "{raw}"); - assert_eq!(resp["status"]["provider"], "managed"); - assert_eq!(resp["status"]["slug"], "managed"); + // The legacy name aliases through to what it now means. + assert_eq!(resp["status"]["provider"], "openrouter"); + assert_eq!(resp["status"]["slug"], "openrouter"); assert_eq!(resp["status"]["source"], "runtime"); assert_eq!(resp["status"]["keyConfigured"], true); - // Setting only a key must not move the tenant off the managed endpoint. + // A key means the tenant's own OpenRouter account pays. assert_eq!( resp["status"]["baseUrl"], - crate::company::inference::MANAGED_BASE_URL + crate::company::inference::OPENROUTER_BASE_URL ); assert!(!raw.contains(TOKEN), "PUT leaked the token: {raw}"); diff --git a/src/server/ops/skills.rs b/src/server/ops/skills.rs index d75e83efc..91fa232da 100644 --- a/src/server/ops/skills.rs +++ b/src/server/ops/skills.rs @@ -34,6 +34,19 @@ const DEFAULT_CATEGORY: &str = "Ops"; /// The publisher stamped on shared-library skills (mirrors the GraphQL type). const REGISTRY_PUBLISHER: &str = "OpenCompany"; +/// Whether `slug` is a safe skill id: `^[a-z0-9][a-z0-9-]*$`. A slug is also a +/// directory name in the agent's scratch tree (`skills//`), so a +/// traversal (`..`) or a path separator here would escape it. Mirrors +/// `harness::built_in::skills::valid_slug`. +fn valid_slug(slug: &str) -> bool { + let mut chars = slug.chars(); + match chars.next() { + Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {} + _ => return false, + } + chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') +} + /// Builds the skills route fragment. pub fn router() -> Router { scoped("/skills/{slug}/install", post(install)) @@ -296,6 +309,12 @@ async fn install( Path(SlugPath { slug }): Path, body: Option>, ) -> Result, ApiError> { + if !valid_slug(&slug) { + return Err(ApiError(OpenCompanyError::InvalidRequest(format!( + "`{slug}` is not a valid skill slug. Skills live under `skills//`, so a slug \ + is `[a-z0-9][a-z0-9-]*`." + )))); + } let registry = state.shared_skill_registry()?; let doc = match registry.iter().find(|doc| doc.slug == slug) { Some(doc) => render_skill_md(doc), @@ -381,6 +400,12 @@ async fn set_enabled( Path(SlugPath { slug }): Path, Json(body): Json, ) -> Result, ApiError> { + if !valid_slug(&slug) { + return Err(ApiError(OpenCompanyError::InvalidRequest(format!( + "`{slug}` is not a valid skill slug. Skills live under `skills//`, so a slug \ + is `[a-z0-9][a-z0-9-]*`." + )))); + } // Preserve an existing delta's source and custom doc; a first toggle of a // built-in company skill records a Company-sourced override. let existing = company @@ -593,4 +618,187 @@ mod tests { assert_eq!(out[0].id, "web-research"); assert_eq!(out[0].source, SkillSource::Registry); } + + /// `valid_slug` is the gate both write handlers share: a slug is also a + /// directory name under `skills//`, so a traversal (`..`) or a path + /// separator (`/`) must never reach the filesystem, and the alphabet is + /// lowercase-only. The path extractor can only ever hand a handler a single + /// segment, so `a/b` cannot arrive as a path — but the function is the + /// contract every slug-bearing caller routes through, so it is the right + /// place to pin all three shapes the review named. + #[test] + fn valid_slug_rejects_traversal_separator_and_case() { + // The shapes the review named. + assert!(!valid_slug(".."), "parent traversal"); + assert!(!valid_slug("a/b"), "path separator"); + assert!(!valid_slug("A"), "uppercase start"); + // And the rest of the boundary. + assert!(!valid_slug(""), "empty"); + assert!(!valid_slug("-leading"), "leading dash"); + assert!(!valid_slug("has space"), "interior space"); + assert!( + !valid_slug("under_score"), + "underscore is not in the alphabet" + ); + assert!(!valid_slug("UPPER"), "all uppercase"); + // And the shape that must pass. + assert!(valid_slug("a-1"), "lowercase, digit, dash"); + assert!(valid_slug("0"), "single digit"); + assert!(valid_slug("seo-audit"), "typical slug"); + } + + /// HTTP-level coverage of the two path-slug handlers. A slug that fails + /// `valid_slug` must be rejected with `400` **before** any write, so the + /// effective skill set is untouched; a valid slug succeeds and lands. + /// + /// `..` and `a/b` cannot be carried as a single path segment (a `/` splits + /// them, and `..` is normalized away by the router), so their rejection is + /// pinned in [`valid_slug_rejects_traversal_separator_and_case`]. `A` is a + /// single segment the router will pass through, so it is the shape we drive + /// through the handlers to prove the `400` and the no-mutation guarantee. + mod http { + use axum::body::{Body, to_bytes}; + use axum::http::{Request, StatusCode}; + use serde_json::Value; + use tower::ServiceExt; + + use crate::company::CompanyManifest; + use crate::ports::CompanyStore; + use crate::ports::types::{CompanyId, CompanyRecord}; + use crate::runtime::RuntimeBuilder; + use crate::server::router; + use crate::server::test_support::{fixed_cookie, seed_fixed_admin}; + use crate::{AppConfig, AppState}; + + async fn state_with_company(home: &std::path::Path) -> AppState { + let id = CompanyId::new("acme"); + let manifest: CompanyManifest = + toml::from_str("[company]\nname = \"Acme\"\n[policy]\nmode = \"full\"\n").unwrap(); + crate::store::FsCompanyStore::new(home.to_path_buf()) + .save(&CompanyRecord { + id: id.clone(), + manifest: manifest.clone(), + ledger: Vec::new(), + lifecycle: "running".to_string(), + overlay_agents: Vec::new(), + overlay_desk_members: Vec::new(), + overlay_desk_order: Vec::new(), + overlay_desks: Vec::new(), + overlay_workflows: Vec::new(), + overlay_budgets: Vec::new(), + overlay_policy: None, + overlay_desk_tools: Default::default(), + disabled_workflows: Vec::new(), + template_provenance: None, + setup: None, + }) + .await + .unwrap(); + let runtime = RuntimeBuilder::new(home.to_path_buf(), manifest) + .with_id(id.clone()) + .build() + .await + .unwrap(); + let state = AppState::new(AppConfig::default()); + state.registry().insert(id, std::sync::Arc::new(runtime)); + seed_fixed_admin(&state, "acme").await; + state + } + + async fn send( + state: &AppState, + method: &str, + uri: &str, + body: Option<&str>, + ) -> (StatusCode, Value, String) { + let request = Request::builder() + .method(method) + .uri(uri) + .header("cookie", fixed_cookie("acme")); + let request = match body { + Some(body) => request + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + None => request.body(Body::empty()).unwrap(), + }; + let response = router(state.clone()).oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let raw = String::from_utf8_lossy(&bytes).to_string(); + let value = if bytes.is_empty() { + Value::Null + } else { + serde_json::from_slice(&bytes).unwrap_or(Value::Null) + }; + (status, value, raw) + } + + /// The effective skill set, as the console reads it. + async fn slugs(state: &AppState) -> Vec { + let (status, value, raw) = send(state, "GET", "/api/v1/company/skills", None).await; + assert_eq!(status, StatusCode::OK, "list skills: {raw}"); + value + .as_array() + .expect("skills list is an array") + .iter() + .map(|s| s["id"].as_str().expect("an id").to_string()) + .collect() + } + + /// Both write handlers reject an invalid slug with `400` and leave the + /// effective skill set untouched; a valid slug then succeeds and lands. + #[tokio::test] + async fn invalid_slugs_are_400_and_leave_state_unchanged() { + let home = tempfile::tempdir().unwrap(); + let state = state_with_company(home.path()).await; + + let before = slugs(&state).await; + + // `install` rejects the uppercase slug without writing. + let (status, _, raw) = + send(&state, "POST", "/api/v1/company/skills/A/install", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "install A: {raw}"); + assert!( + raw.contains("not a valid skill slug"), + "the 400 explains why: {raw}" + ); + + // `set_enabled` rejects the same slug without writing. + let (status, _, raw) = send( + &state, + "PUT", + "/api/v1/company/skills/A", + Some(r#"{"enabled":true}"#), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "set_enabled A: {raw}"); + + // Neither attempt mutated the effective set. + assert_eq!( + slugs(&state).await, + before, + "a rejected slug must not land a delta" + ); + + // A valid slug succeeds on both handlers and does land. + let (status, _, raw) = + send(&state, "POST", "/api/v1/company/skills/a-1/install", None).await; + assert_eq!(status, StatusCode::OK, "install a-1: {raw}"); + + let (status, _, raw) = send( + &state, + "PUT", + "/api/v1/company/skills/a-1", + Some(r#"{"enabled":false}"#), + ) + .await; + assert_eq!(status, StatusCode::OK, "set_enabled a-1: {raw}"); + + assert!( + slugs(&state).await.iter().any(|s| s == "a-1"), + "the valid slug lands in the effective set" + ); + } + } } diff --git a/src/server/ops/write_test.rs b/src/server/ops/write_test.rs index 46dee0d12..e7a32ecbd 100644 --- a/src/server/ops/write_test.rs +++ b/src/server/ops/write_test.rs @@ -2424,6 +2424,57 @@ async fn skills_registry_is_empty_when_no_library_is_served() { assert_eq!(body.as_array().expect("an array").len(), 0); } +/// A slug is also a directory name (`skills//`), so the handlers that +/// take one from the URL must refuse the values that could escape or traverse +/// the scratch tree: a parent segment (`..`), a path separator (`a/b`), and a +/// leading uppercase (`A` — slugs are lowercase by contract). +#[tokio::test] +async fn skill_handlers_reject_unsafe_slugs_and_write_nothing() { + let home_dir = home(); + let state = state_with_registry(home_dir.path()).await; + + // `a%2Fb` is how a `/` arrives *inside* one path segment — the router sees + // one slug and the handler must reject it rather than letting a separator + // into a directory name. + for bad in ["..", "a%2Fb", "A"] { + let (status, body) = send( + &state, + "POST", + &format!("/api/v1/company/skills/{bad}/install"), + Some(json!({"name": "Name", "description": "desc"})), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "install {bad}: {body}"); + + let (status, body) = send( + &state, + "PUT", + &format!("/api/v1/company/skills/{bad}"), + Some(json!({"enabled": true})), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "toggle {bad}: {body}"); + } + + // A rejected slug must never reach the skill store. + assert!( + persisted_skills(&state).await.is_empty(), + "an unsafe slug must not persist a delta" + ); + + // The same handlers still accept a well-formed slug. + let (status, skill) = send( + &state, + "PUT", + "/api/v1/company/skills/a-1", + Some(json!({"enabled": true})), + ) + .await; + assert_eq!(status, StatusCode::OK, "{skill}"); + assert_eq!(skill["id"], "a-1"); + assert_eq!(skill["enabled"], true); +} + #[tokio::test] async fn skills_install_toggle_custom_and_builtin_uninstall_conflict() { let home_dir = home(); diff --git a/src/workflows/blocked_node_continuation_test.rs b/src/workflows/blocked_node_continuation_test.rs index 29174ade3..2ab06e67d 100644 --- a/src/workflows/blocked_node_continuation_test.rs +++ b/src/workflows/blocked_node_continuation_test.rs @@ -206,8 +206,14 @@ async fn runtime( let pool = Arc::new(HarnessPool::new()); pool.ensure(&record(), &deps).await.expect("roster builds"); + // Single-harness fixture: the default lane over the pool is the turn + // (mirrors `run_workflow`'s single-pool entrypoint). + let turn = Arc::new(crate::harness::built_in::run_turn::HarnessRunTurn::new( + pool, + Arc::new(deps.clone()), + )); let runner = Arc::new(RecordingRunner { - inner: super::runner::HarnessWorkflowRunner::new(pool, deps, record()), + inner: super::runner::HarnessWorkflowRunner::new(turn, deps, record()), started: Mutex::new(Vec::new()), }); rt.set_workflow_runner(runner.clone()); diff --git a/src/workflows/caps/mod.rs b/src/workflows/caps/mod.rs index a53b05061..6c9c3a1e9 100644 --- a/src/workflows/caps/mod.rs +++ b/src/workflows/caps/mod.rs @@ -70,8 +70,9 @@ use tinyflows::error::{EngineError, Result as TfResult}; use crate::harness::orchestrator::MAX_DELEGATIONS_PER_TURN; use crate::harness::policy::{ApprovalScope, MAX_APPROVAL_REQUESTS_PER_TURN, PolicyMode}; -use crate::harness::{HarnessDeps, HarnessPool, toolbelt}; +use crate::harness::{HarnessDeps, toolbelt}; use crate::ports::types::{CompanyId, CompanyRecord}; +use crate::runtime::delegation::RunTurn; use self::http::GuardedHttpClient; use self::resolver::StoreWorkflowResolver; @@ -141,8 +142,8 @@ pub struct RunContext<'a> { /// `_` prefix keeps it from ever colliding with a roster agent's own workspace /// directory. /// -/// `pool`/`deps` are shared with the rest of the harness surface — the roster the -/// agent nodes address is the one already resident in `pool`. +/// `turn`/`deps` are shared with the rest of the harness surface — the roster the +/// agent nodes address is the one resident in the harness(es) the turn routes to. /// /// `run_request` is the operator's topic for this run (issue #154), threaded to /// the agent capability so every agent node's turn message carries what was @@ -168,7 +169,7 @@ pub struct RunContext<'a> { /// rather than proceeding with effects pointed at a directory that does not /// exist. A dry run builds no workspace and is infallible. pub async fn build_capabilities( - pool: Arc, + turn: Arc, deps: HarnessDeps, record: &CompanyRecord, run: RunContext<'_>, @@ -345,7 +346,7 @@ pub async fn build_capabilities( // `deps.search`, `deps.meter`, `deps.secrets`, `deps.workspace_root`, // `deps.delegations`) are all done by here. let agent: Arc = Arc::new(HarnessAgentRunner::new( - pool, + turn, deps, record.clone(), company.clone(), @@ -475,7 +476,10 @@ fn hex_segment(value: &str) -> String { /// `publish_artifact` needs a card to attach a version to, which a run does not /// have, so a refusal there remains the truthful answer. pub struct HarnessAgentRunner { - pool: Arc, + /// The turn a workflow agent node runs on: the lane-aware router in a + /// multi-harness company, the default lane over the pool in a + /// single-harness one (see `run_workflow`'s single-pool entrypoint). + turn: Arc, deps: HarnessDeps, /// The company record, for the board drain's desk/assignee resolution (issue /// #661 / M5) — the same record the rest of this bundle was built from, so a @@ -675,12 +679,14 @@ impl ParkedCalls { } impl HarnessAgentRunner { - /// Builds a runner over an already-populated pool for `company`, carrying - /// the run's id (issue #395) and the operator's run request (issue #154) - /// when one was supplied. + /// Builds a runner over `turn` for `company`, carrying the run's id (issue + /// #395) and the operator's run request (issue #154) when one was supplied. + /// The turn is the lane-aware router where lanes are declared, or the + /// default lane over the pool otherwise, so a workflow agent node addressing + /// a named-harness agent reaches that harness's engine. #[allow(clippy::too_many_arguments)] pub fn new( - pool: Arc, + turn: Arc, deps: HarnessDeps, record: CompanyRecord, company: CompanyId, @@ -694,7 +700,7 @@ impl HarnessAgentRunner { board_claim: Arc, ) -> Self { Self { - pool, + turn, deps, record, company, @@ -1139,7 +1145,7 @@ impl AgentRunner for HarnessAgentRunner { tracing::debug!( company = %self.company, agent = agent_ref, - "workflow agent node: routing to harness pool" + "workflow agent node: routing through harness turn" ); // Issue #439: this run's own approval scope, replacing #395's boundary // index. The index was only ever a narrowing — it was taken against a @@ -1168,11 +1174,10 @@ impl AgentRunner for HarnessAgentRunner { // first spawning run without these. let turn = Box::pin(async { let outcome = claim - .scoped(Box::pin(self.pool.run_background( + .scoped(Box::pin(self.turn.run_background( &self.company, agent_ref, &message, - &self.deps, ))) .await; // Drained on BOTH arms, deliberately. A turn that errored may still have @@ -1606,6 +1611,15 @@ impl CodeRunner for UnwiredCode { mod tests { use super::*; + /// The single-harness turn over a fresh pool, as the non-lane entrypoint + /// wraps — what a workflow agent node runs on when no lanes are declared. + fn single_turn(deps: &HarnessDeps) -> Arc { + Arc::new(crate::harness::built_in::run_turn::HarnessRunTurn::new( + Arc::new(crate::harness::HarnessPool::new()), + Arc::new(deps.clone()), + )) + } + /// Issue #638: a node that gates more calls than the cap allows leaves the /// operator a **notice**, not only a log line. /// @@ -1683,7 +1697,7 @@ mod tests { let notices = RunNotices::default(); let board_claim = Arc::new(deps.delegations.claim_board("run-1")); let runner = HarnessAgentRunner::new( - Arc::new(HarnessPool::new()), + single_turn(&deps), deps, crate::workflows::gated_tool_turn_test::record(), CompanyId::new("acme"), @@ -2230,7 +2244,7 @@ mod tests { let record = crate::workflows::gated_tool_turn_test::record(); let caps = build_capabilities( - Arc::new(HarnessPool::new()), + single_turn(&deps), deps, &record, RunContext { @@ -2276,7 +2290,7 @@ mod tests { let record = crate::workflows::gated_tool_turn_test::record(); let caps = build_capabilities( - Arc::new(HarnessPool::new()), + single_turn(&deps), deps, &record, RunContext { @@ -2439,7 +2453,7 @@ mod tests { // `Capabilities` is not `Debug`, so match rather than `expect_err`. let err = match build_capabilities( - Arc::new(HarnessPool::new()), + single_turn(&deps), deps, &record, RunContext { @@ -2492,7 +2506,7 @@ mod tests { let record = crate::workflows::gated_tool_turn_test::record(); build_capabilities( - Arc::new(HarnessPool::new()), + single_turn(&deps), deps, &record, RunContext { diff --git a/src/workflows/gated_tool_turn_test.rs b/src/workflows/gated_tool_turn_test.rs index dda0ac588..2f3ce1a69 100644 --- a/src/workflows/gated_tool_turn_test.rs +++ b/src/workflows/gated_tool_turn_test.rs @@ -209,6 +209,7 @@ pub(super) fn deps(base_url: String, dir: &std::path::Path) -> (HarnessDeps, Arc extra_headers: Vec::new(), })), provider_slug: "managed".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: None, diff --git a/src/workflows/parallel_gate_fanout_test.rs b/src/workflows/parallel_gate_fanout_test.rs index 3cba2e716..9e5a336ba 100644 --- a/src/workflows/parallel_gate_fanout_test.rs +++ b/src/workflows/parallel_gate_fanout_test.rs @@ -260,8 +260,14 @@ async fn runtime( let pool = Arc::new(crate::harness::HarnessPool::new()); pool.ensure(&record(), &deps).await.expect("roster builds"); + // Single-harness fixture: the default lane over the pool is the turn + // (mirrors `run_workflow`'s single-pool entrypoint). + let turn = Arc::new(crate::harness::built_in::run_turn::HarnessRunTurn::new( + pool, + Arc::new(deps.clone()), + )); let runner = Arc::new(RecordingRunner { - inner: super::runner::HarnessWorkflowRunner::new(pool, deps, record()), + inner: super::runner::HarnessWorkflowRunner::new(turn, deps, record()), started: Mutex::new(Vec::new()), }); rt.set_workflow_runner(runner.clone()); diff --git a/src/workflows/runner.rs b/src/workflows/runner.rs index 0d303cb1c..8363cca83 100644 --- a/src/workflows/runner.rs +++ b/src/workflows/runner.rs @@ -93,6 +93,28 @@ pub async fn run_workflow( workflow: &WorkflowFile, input: Value, ctx: &WorkflowRunContext, +) -> Result { + // A single pool is the single-harness case: the router is just the default + // lane over that pool. Kept as its own entrypoint so the many single-pool + // tests (and any single-harness caller) need not hand-assemble a router. + let turn: Arc = Arc::new( + crate::harness::built_in::run_turn::HarnessRunTurn::new(pool, Arc::new(deps.clone())), + ); + run_workflow_lane_aware(turn, deps, record, workflow, input, ctx).await +} + +/// [`run_workflow`] over an already-assembled, lane-aware router. +/// +/// The production [`HarnessWorkflowRunner`] takes this path so a workflow +/// `agent` node addressing a named-harness agent routes to that harness's +/// engine instead of the default pool. +pub async fn run_workflow_lane_aware( + turn: Arc, + deps: HarnessDeps, + record: &CompanyRecord, + workflow: &WorkflowFile, + input: Value, + ctx: &WorkflowRunContext, ) -> Result { // Issue #151 part a: refuse an unbounded re-entry before it takes the host // down. `run_workflow` is an orchestrator tool, and a workflow `agent` node @@ -122,7 +144,7 @@ pub async fn run_workflow( WORKFLOW_DEPTH .scope( depth + 1, - run_workflow_inner(pool, deps, record, workflow, input, ctx), + run_workflow_inner(turn, deps, record, workflow, input, ctx), ) .await } @@ -241,7 +263,7 @@ impl tinyflows::observability::RunObserver for ProgressObserver { /// The run itself, always executed inside a [`WORKFLOW_DEPTH`] scope so a /// nested run sees this one on the chain. async fn run_workflow_inner( - pool: Arc, + turn: Arc, deps: HarnessDeps, record: &CompanyRecord, workflow: &WorkflowFile, @@ -379,7 +401,7 @@ async fn run_workflow_inner( let blocks = super::caps::RunBlocks::default(); let approvals = super::caps::RunApprovals::default(); let capabilities = super::caps::build_capabilities( - pool, + turn, deps, record, super::caps::RunContext { @@ -1676,19 +1698,24 @@ fn map_engine_error(err: tinyflows::error::EngineError) -> OpenCompanyError { } /// The [`WorkflowRunner`] port backed by the embedded harness: it holds the -/// shared pool, its deps, and the company record so it can ensure the roster is -/// built before a run and route agent nodes onto it. +/// lane-aware router so every agent node dispatches to the engine its harness +/// demands, plus the deps and company record it needs to warm every lane before +/// a run. pub struct HarnessWorkflowRunner { - pool: Arc, + turn: Arc, deps: HarnessDeps, record: CompanyRecord, } impl HarnessWorkflowRunner { - /// Builds a runner sharing `pool`/`deps` with the rest of the harness surface - /// for the company described by `record`. - pub fn new(pool: Arc, deps: HarnessDeps, record: CompanyRecord) -> Self { - Self { pool, deps, record } + /// Builds a runner dispatching through `turn`, sharing `deps` with the + /// rest of the harness surface for the company described by `record`. + pub fn new( + turn: Arc, + deps: HarnessDeps, + record: CompanyRecord, + ) -> Self { + Self { turn, deps, record } } } @@ -1701,12 +1728,14 @@ impl WorkflowRunner for HarnessWorkflowRunner { input: Value, ctx: &WorkflowRunContext, ) -> Result { - // Idempotent: builds the roster on first use, a no-op after. The run - // addresses the record's own company; `_company` is the routed scope, - // which the runtime resolves to this same record. - self.pool.ensure(&self.record, &self.deps).await?; - run_workflow( - self.pool.clone(), + // Idempotent: builds the roster on first use, a no-op after. Warmed + // through the router so every lane's pool — not just the default's — is + // populated before a node addresses it. The run addresses the record's + // own company; `_company` is the routed scope, which the runtime + // resolves to this same record. + self.turn.ensure(&self.record).await?; + run_workflow_lane_aware( + self.turn.clone(), self.deps.clone(), &self.record, workflow, @@ -1726,6 +1755,64 @@ mod tests { use crate::ports::run_output::WorkflowRunOutputStore; use crate::store::{FsCompanyStore, FsContextStore, FsOps}; + /// A workflow lane that records which agent it served. Its reply names the + /// lane so the run output proves the same routing decision as the call log. + struct RecordingLane { + label: &'static str, + seen: std::sync::Mutex>, + } + + impl RecordingLane { + fn new(label: &'static str) -> Arc { + Arc::new(Self { + label, + seen: std::sync::Mutex::new(Vec::new()), + }) + } + } + + #[async_trait] + impl crate::runtime::delegation::RunTurn for RecordingLane { + async fn run( + &self, + _company: &CompanyId, + agent_id: &str, + _message: &str, + _chat_id: Option<&str>, + ) -> Result { + self.seen.lock().unwrap().push(agent_id.to_string()); + Ok(crate::harness::TurnOutcome { + reply: self.label.to_string(), + steps: Vec::new(), + hit_iteration_cap: false, + halted_for_spend: None, + }) + } + + async fn run_steered( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + _control: &crate::company::steer::SteerControl, + chat_id: Option<&str>, + _run_sink: Option>, + ) -> Result { + self.run(company, agent_id, message, chat_id).await + } + + async fn run_steered_background( + &self, + company: &CompanyId, + agent_id: &str, + message: &str, + _control: &crate::company::steer::SteerControl, + _run_sink: Option>, + ) -> Result { + self.run(company, agent_id, message, None).await + } + } + fn record() -> CompanyRecord { let manifest = toml::from_str( r#" @@ -1768,6 +1855,7 @@ description = "Runs Acme." run_supervisor: crate::runtime::RunSupervisor::default(), provider: Arc::new(MockProvider::new("mock: ")), provider_slug: "mock".to_string(), + serves: None, context: Arc::new(FsContextStore::new(dir)), store: Arc::new(FsCompanyStore::new(dir)), meter: Some(Arc::new(FsOps::new(dir))), @@ -2587,7 +2675,12 @@ to = "done" let dir = tempfile::tempdir().unwrap(); let pool = Arc::new(HarnessPool::new()); let rec = record(); - let runner = HarnessWorkflowRunner::new(pool, deps(dir.path()), rec.clone()); + let deps = deps(dir.path()); + let turn = Arc::new(crate::harness::built_in::run_turn::HarnessRunTurn::new( + pool, + Arc::new(deps.clone()), + )); + let runner = HarnessWorkflowRunner::new(turn, deps, rec.clone()); let file = parse_workflow(GREET).expect("workflow parses"); let run = WorkflowRunner::run( @@ -2602,6 +2695,39 @@ to = "done" assert!(run.output.to_string().contains("hello-marker")); } + /// The workflow port keeps the lane-aware router intact: an agent bound to + /// a named harness must not fall back to the default engine. + #[tokio::test] + async fn port_impl_routes_an_agent_node_to_its_named_harness() { + let dir = tempfile::tempdir().unwrap(); + let rec = record(); + let deps = deps(dir.path()); + let default = RecordingLane::new("default-lane"); + let deep = RecordingLane::new("deep-lane"); + let turn: Arc = Arc::new( + crate::harness::router::HarnessRouter::new("embedded") + .with_engine("embedded", default.clone()) + .with_engine("deep", deep.clone()) + .bind("ceo", "deep"), + ); + let runner = HarnessWorkflowRunner::new(turn, deps, rec.clone()); + + let file = parse_workflow(GREET).expect("workflow parses"); + let run = WorkflowRunner::run( + &runner, + &rec.id, + &file, + serde_json::json!({}), + &WorkflowRunContext::new(false), + ) + .await + .expect("workflow runs through the named lane"); + + assert!(run.output.to_string().contains("deep-lane")); + assert!(default.seen.lock().unwrap().is_empty()); + assert_eq!(&*deep.seen.lock().unwrap(), &["ceo".to_string()]); + } + /// A workflow with no trigger is a caller-facing bad request, not a harness /// error. (Built by hand — `parse_workflow` would reject it earlier.) #[tokio::test] diff --git a/src/workflows/workflow_standing_grant_test.rs b/src/workflows/workflow_standing_grant_test.rs index a8015abf4..e87a77cef 100644 --- a/src/workflows/workflow_standing_grant_test.rs +++ b/src/workflows/workflow_standing_grant_test.rs @@ -394,7 +394,10 @@ async fn runtime(home: &std::path::Path) -> Arc