diff --git a/.beads/.gitignore b/.beads/.gitignore new file mode 100644 index 00000000..4176bee9 --- /dev/null +++ b/.beads/.gitignore @@ -0,0 +1,76 @@ +# Dolt database (managed by Dolt, not git) +dolt/ +embeddeddolt/ +proxieddb/ + +# Runtime files +bd.sock +bd.sock.startlock +sync-state.json +last-touched +.exclusive-lock + +# Daemon runtime (lock, log, pid) +daemon.* + +# Push state (runtime, per-machine) +push-state.json + +# Lock files (various runtime locks) +*.lock + +# Credential key (encryption key for federation peer auth — never commit) +.beads-credential-key + +# Local version tracking (prevents upgrade notification spam after git ops) +.local_version + +proxied_server_client_info.json + +# Worktree redirect file (contains relative path to main repo's .beads/) +# Must not be committed as paths would be wrong in other clones +redirect + +# Sync state (local-only, per-machine) +# These files are machine-specific and should not be shared across clones +.sync.lock +export-state/ +export-state.json + +# Ephemeral store (SQLite - wisps/molecules, intentionally not versioned) +ephemeral.sqlite3 +ephemeral.sqlite3-journal +ephemeral.sqlite3-wal +ephemeral.sqlite3-shm + +# Dolt server management (auto-started by bd) +dolt-server.pid +dolt-server.log +dolt-server.lock +dolt-server.port +dolt-server.activity + +# Debug-mode pprof artifacts (written when dolt.debug: true in config.yaml) +dolt-pprof/ + +# Corrupt backup directories (created by bd doctor --fix recovery) +*.corrupt.backup/ + +# Backup data (auto-exported JSONL, local-only) +backup/ + +# Per-project environment file (Dolt connection config, GH#2520) +.env + +# Legacy files (from pre-Dolt versions) +*.db +*.db?* +*.db-journal +*.db-wal +*.db-shm +db.sqlite +bd.db +# NOTE: Do NOT add negation patterns here. +# They would override fork protection in .git/info/exclude. +# Config files (metadata.json, config.yaml) are tracked by git by default +# since no pattern above ignores them. diff --git a/.beads/README.md b/.beads/README.md new file mode 100644 index 00000000..63e8f4c2 --- /dev/null +++ b/.beads/README.md @@ -0,0 +1,81 @@ +# Beads - AI-Native Issue Tracking + +Welcome to Beads! This repository uses **Beads** for issue tracking - a modern, AI-native tool designed to live directly in your codebase alongside your code. + +## What is Beads? + +Beads is issue tracking that lives in your repo, making it perfect for AI coding agents and developers who want their issues close to their code. No web UI required - everything works through the CLI and integrates seamlessly with git. + +**Learn more:** [github.com/steveyegge/beads](https://github.com/steveyegge/beads) + +## Quick Start + +### Essential Commands + +```bash +# Create new issues +bd create "Add user authentication" + +# View all issues +bd list + +# View issue details +bd show + +# Update issue status +bd update --claim +bd update --status done + +# Sync with Dolt remote +bd dolt push +``` + +### Working with Issues + +Issues in Beads are: +- **Git-native**: Stored in Dolt database with version control and branching +- **AI-friendly**: CLI-first design works perfectly with AI coding agents +- **Branch-aware**: Issues can follow your branch workflow +- **Sync-ready**: Uses Dolt remotes for backup and team sharing + +## Why Beads? + +✨ **AI-Native Design** +- Built specifically for AI-assisted development workflows +- CLI-first interface works seamlessly with AI coding agents +- No context switching to web UIs + +🚀 **Developer Focused** +- Issues live in your repo, right next to your code +- Works offline, syncs when you push +- Fast, lightweight, and stays out of your way + +🔧 **Git Integration** +- Dolt-native sync via bd dolt push / bd dolt pull +- Branch-aware issue tracking +- Dolt-native three-way merge resolution + +## Get Started with Beads + +Try Beads in your own projects: + +```bash +# Install Beads +curl -sSL https://raw.githubusercontent.com/steveyegge/beads/main/scripts/install.sh | bash + +# Initialize in your repo +bd init + +# Create your first issue +bd create "Try out Beads" +``` + +## Learn More + +- **Documentation**: [github.com/steveyegge/beads/docs](https://github.com/steveyegge/beads/tree/main/docs) +- **Quick Start Guide**: Run `bd quickstart` +- **Examples**: [github.com/steveyegge/beads/examples](https://github.com/steveyegge/beads/tree/main/examples) + +--- + +*Beads: Issue tracking that moves at the speed of thought* ⚡ diff --git a/.beads/config.yaml b/.beads/config.yaml new file mode 100644 index 00000000..b65eb45e --- /dev/null +++ b/.beads/config.yaml @@ -0,0 +1,68 @@ +# Beads Configuration File +# This file configures default behavior for all bd commands in this repository +# All settings can also be set via environment variables (BD_* prefix) +# or overridden with command-line flags + +# Issue prefix for this repository (used by bd init) +# If not set, bd init will auto-detect from directory name +# Example: issue-prefix: "myproject" creates issues like "myproject-1", "myproject-2", etc. +# issue-prefix: "" + +# Use no-db mode: JSONL-only, no Dolt database +# When true, .beads/issues.jsonl is the only local store +# no-db: false + +# Enable JSON output by default +# json: false + +# Feedback title formatting for mutating commands (create/update/close/dep/edit) +# 0 = hide titles, N > 0 = truncate to N characters +# output: +# title-length: 255 + +# Default actor for audit trails (overridden by BEADS_ACTOR or --actor) +# actor: "" + +# Export events (audit trail) to .beads/events.jsonl on each flush/sync +# When enabled, new events are appended incrementally using a high-water mark. +# Use 'bd export --events' to trigger manually regardless of this setting. +# events-export: false + +# Multi-repo configuration (experimental - bd-307) +# Allows hydrating from multiple repositories and routing writes to the correct database +# repos: +# primary: "." # Primary repo (where this database lives) +# additional: # Additional repos to hydrate from (read-only) +# - ~/beads-planning # Personal planning repo +# - ~/work-planning # Work planning repo + +# Dolt-native backup (periodic backup for off-machine recovery) +# This is full database backup only. Cross-machine sync uses Dolt remotes. +# backup: +# enabled: false # Disable auto-backup entirely +# interval: 15m # Minimum time between auto-backups +# git-push: false # Disable git push (backup locally only) +# git-repo: "" # Separate git repo for backups (default: project repo) + +# Optional JSONL auto-export for viewers, interchange, and issue-level migration. +# Disabled by default; enable only when an integration needs fresh .beads/issues.jsonl. +# Use relative paths under .beads/ for JSONL import/export filenames. +# export: +# auto: false +# path: issues.jsonl +# interval: 60s +# git-add: false +# import: +# path: issues.jsonl + +# Integration settings (access with 'bd config get/set') +# Non-secret keys (stored in the database): +# - jira.url, jira.project +# - linear.team_id +# - github.org, github.repo +# +# Secret keys (stored in this file but prefer env vars to avoid git exposure): +# - linear.api_key → use LINEAR_API_KEY env var instead +# - github.token → use GITHUB_TOKEN env var instead + +sync.remote: "git+https://github.com/DaveGerson/agent-baton.git" \ No newline at end of file diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl new file mode 100644 index 00000000..aa59eb71 --- /dev/null +++ b/.beads/interactions.jsonl @@ -0,0 +1,33 @@ +{"id":"int-edacb656b528e95389cb915514297234","kind":"field_change","created_at":"2026-06-30T03:57:41.8849787Z","actor":"gerso","issue_id":"bd-rm-plan-p1","extra":{"field":"status","new_value":"in_progress","old_value":"open"}} +{"id":"int-84ef1c18ffd6ff08e4867ac0ac4c7227","kind":"field_change","created_at":"2026-06-30T03:57:43.1817056Z","actor":"gerso","issue_id":"bd-rm-knowledge-p1","extra":{"field":"status","new_value":"in_progress","old_value":"open"}} +{"id":"int-dd7f40d2575e1458fce685921fd53896","kind":"field_change","created_at":"2026-06-30T05:33:36.1828691Z","actor":"gerso","issue_id":"bd-rm-plan-p1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Foundation Phase 1 slice implemented and verified: planning diagnostics/knowledge auto-load active across planner CLI API PMO and SQLite; final adversarial review clean; pytest gates passed."}} +{"id":"int-10be8fc866ded03d20d1711b11d2ad4d","kind":"field_change","created_at":"2026-06-30T05:33:36.4904019Z","actor":"gerso","issue_id":"bd-rm-knowledge-p1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Foundation Phase 1 slice implemented and verified: planning diagnostics/knowledge auto-load active across planner CLI API PMO and SQLite; final adversarial review clean; pytest gates passed."}} +{"id":"int-c0d7018d00b4afe6eb079457203ca9ce","kind":"field_change","created_at":"2026-07-02T01:13:37.2758662Z","actor":"gerso","issue_id":"bd-rm-plan-p2","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented actionable plan quality gates with default critical blocking, hard-gate precedence, actionable remediation, review/audit fail-closed validation, strict golden snapshots, clean adversarial review, and fresh verification: plan-quality 22 passed; validation/review/classifier 36 passed; git diff --check clean."}} +{"id":"int-3e1466ef944daff622f58e592affd18f","kind":"field_change","created_at":"2026-07-02T01:13:37.9452641Z","actor":"gerso","issue_id":"bd-rm-knowledge-p2","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented knowledge doctor/search/resolve CLI with cooperative parser normalization, actionable doctor JSON/strict output, resolver simulation, clean adversarial review, and fresh verification: knowledge 75 passed; CLI compatibility 58 passed; git diff --check clean."}} +{"id":"int-cf7c1df1a119c275a40867e17c7aa0cf","kind":"field_change","created_at":"2026-07-02T01:57:20.0352149Z","actor":"gerso","issue_id":"bd-rm-team-p1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented team readiness diagnostics, strict backend mode, Claude Teams caveats, and team report artifacts with clean adversarial review. Fresh verification: team suite 40 passed; strict backend/API checks 3 passed with existing Starlette warning; git diff --check clean."}} +{"id":"int-20f2eec6987f5d50417e9cdb24cf315d","kind":"field_change","created_at":"2026-07-02T01:57:20.58754Z","actor":"gerso","issue_id":"bd-rm-talent-p1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented generated-agent contract in Talent Builder, starter templates, authoring docs, roster alias wording, and parser/contract tests with clean adversarial review. Fresh verification: agent validation 29 valid/1 known warning/0 errors; tests/agents + registry 39 passed; git diff --check clean."}} +{"id":"int-b30f119381f7b0331d74290cf6a7ecdf","kind":"field_change","created_at":"2026-07-02T03:28:11.5453072Z","actor":"gerso","issue_id":"bd-rm-ux-p1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented top-level baton doctor with human/JSON output, terminology cleanup, read-only Beads/worktree/planner/team-context diagnostics, and warning-mode degradation for optional assets. Verified: tests/cli/test_doctor.py 20 passed; baton doctor exit 0; baton doctor --json parsed; CLI help discovery includes doctor; git diff --check clean."}} +{"id":"int-1cbe357426e3de374f20781ee3b35f4a","kind":"field_change","created_at":"2026-07-02T07:56:51.7062844Z","actor":"gerso","issue_id":"bd-fnn","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-6e124152dd8622f004ec14b663875247","kind":"field_change","created_at":"2026-07-02T07:56:51.882806Z","actor":"gerso","issue_id":"bd-xjm","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-031244bb2b74b9aeb86045fd7b50c215","kind":"field_change","created_at":"2026-07-02T07:56:52.0286857Z","actor":"gerso","issue_id":"bd-3ba","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-7df4765cbd87b149d1db7f61d7201595","kind":"field_change","created_at":"2026-07-02T07:56:52.1786893Z","actor":"gerso","issue_id":"bd-45e","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-a28dfa72da4b15bfdc9e46e1505cc9d4","kind":"field_change","created_at":"2026-07-02T07:56:52.3335859Z","actor":"gerso","issue_id":"bd-74e","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-4357c969872b499a38516a587c7029fb","kind":"field_change","created_at":"2026-07-02T07:56:52.4821136Z","actor":"gerso","issue_id":"bd-2c0","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-fc3d790e55fd992b16bf2fd0e1680182","kind":"field_change","created_at":"2026-07-02T07:56:52.626557Z","actor":"gerso","issue_id":"bd-c09","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-5fba6fcf016481217145c0e7eccc24c0","kind":"field_change","created_at":"2026-07-02T07:56:52.7743555Z","actor":"gerso","issue_id":"bd-mjm","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-55e30b35df23007993e0ce1281df1720","kind":"field_change","created_at":"2026-07-02T07:56:52.9288094Z","actor":"gerso","issue_id":"bd-4j0","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-19e8db4b510c4dfe34614d9905dfb4da","kind":"field_change","created_at":"2026-07-02T07:56:53.0817846Z","actor":"gerso","issue_id":"bd-c5g","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-1f9083afdecd92163c807fa8b1b88da5","kind":"field_change","created_at":"2026-07-02T07:56:53.2308121Z","actor":"gerso","issue_id":"bd-sa4","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-ebb56191a26483ac47e6c08e057176a5","kind":"field_change","created_at":"2026-07-02T07:56:53.3758707Z","actor":"gerso","issue_id":"bd-pg5","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-d60e4368aa3e53b96e50ed3bbddf7c36","kind":"field_change","created_at":"2026-07-02T07:56:53.5244906Z","actor":"gerso","issue_id":"bd-3cj","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented in isolated worktree and verified with focused regression suite: 222 passed; git diff --check clean."}} +{"id":"int-40dad5240bea7d15cd4c41d7b6f63903","kind":"field_change","created_at":"2026-07-02T21:51:34.6184975Z","actor":"gerso","issue_id":"bd-8ab","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Wave 0 merged at a0d4236 after Fable review + fix cycle; 98 tests green"}} +{"id":"int-d065f9bce59e92ed406542d886865a73","kind":"field_change","created_at":"2026-07-02T22:40:15.4933902Z","actor":"gerso","issue_id":"bd-vib","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Wave 1 merged at e0c2981 + fix cycle 65ef781; review APPROVED; 431+ tests green; ab_cmd parser fixed with main() E2E guards"}} +{"id":"int-a210cc2814c035dea803cda50b5469e8","kind":"field_change","created_at":"2026-07-02T22:40:15.6738566Z","actor":"gerso","issue_id":"bd-b6i","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Wave 1 merged at e0c2981 + fix cycle 65ef781; review APPROVED; 431+ tests green; ab_cmd parser fixed with main() E2E guards"}} +{"id":"int-a1a1b4c50089433a4a5b37ff8d2f9e35","kind":"field_change","created_at":"2026-07-02T23:28:59.3012762Z","actor":"gerso","issue_id":"bd-tgq","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Wave 2 merged at 0e3831f + fix cycle e20cd64; review CHANGES_REQUIRED resolved (gate rescope fidelity, reviewing step_type, byte-identity snapshot, overflow ordering); 594 tests green"}} +{"id":"int-a64efa3406c6d0bf2a44f72ec52a0bbe","kind":"field_change","created_at":"2026-07-03T00:20:34.6113962Z","actor":"gerso","issue_id":"bd-0ak","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Wave 3 complete: composition ec05288, M8 E2E 169ce94, fix cycle 8ebcf07; live CLI smoke passed; 608 tests green"}} +{"id":"int-8427f59bd05253fc4b8fb3a21bbedbcf","kind":"field_change","created_at":"2026-07-03T00:57:45.1675678Z","actor":"gerso","issue_id":"bd-29o","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-f7e55968c4895c9c53337ac9d4951569","kind":"field_change","created_at":"2026-07-03T01:28:50.5135602Z","actor":"gerso","issue_id":"bd-nep","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Wave 4 complete: M9 76b3c34, docs 27b2902, final review fixes 4b2eb03; release review checklist 25/25 MET post-fix"}} +{"id":"int-4d82052365607b9bd3bbe559e7db2392","kind":"field_change","created_at":"2026-07-03T03:42:55.170456Z","actor":"gerso","issue_id":"bd-t8u","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented on claude/manager-mode-followups (a1d4fad, 6333777), merged into PR #111 (open, awaiting director merge); 702 tests green"}} +{"id":"int-47f29fcd4eb72c055882d94bfa857cc9","kind":"field_change","created_at":"2026-07-03T03:42:55.3794412Z","actor":"gerso","issue_id":"bd-6dn","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented on claude/manager-mode-followups (a1d4fad, 6333777), merged into PR #111 (open, awaiting director merge); 702 tests green"}} +{"id":"int-55233ecb5ca93873ebc945f863041274","kind":"field_change","created_at":"2026-07-03T03:42:56.5445665Z","actor":"gerso","issue_id":"bd-a63","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Investigation complete: 4-conflict map + landing sequence in scratchpad hub-reconcile-report.md; execution requires hub session (31 unpushed commits + 1.9k uncommitted lines)"}} +{"id":"int-b72aacf782dcdef7443894371b161eb6","kind":"field_change","created_at":"2026-07-06T15:51:44.4304982Z","actor":"gerso","issue_id":"bd-dm3","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"All 14 punch-list items resolved: L1 autouse _sandbox_home fixture, L2 quickstart project-root knowledge wiring, L3 --explain+--json JSON payload, L4 dotted-stem .md fallback, L5 expanduser on knowledge roots, L6 strict missing-root issue, L7 knowledge CLI docs section, L8 talent-manager alias claims deleted, L9 posix report_path, L10 dynamic warnings before static caveats, L11 reference count 20 synced, L12 mid-run backend-error semantics documented+tested, L13 golden-test env isolation, L14 writability caveat documented. Full-suite A/B confirmed zero regressions (175 pre-existing Windows-env failures identical at baseline)."}} diff --git a/.beads/metadata.json b/.beads/metadata.json new file mode 100644 index 00000000..8391c32e --- /dev/null +++ b/.beads/metadata.json @@ -0,0 +1,7 @@ +{ + "database": "dolt", + "backend": "dolt", + "dolt_mode": "embedded", + "dolt_database": "bd", + "project_id": "1cea0a5a-9de8-49cb-a746-2f0f056f9725" +} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 10222352..6daa6cb0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,7 +36,7 @@ Apply these to every change before declaring it done: | `agent_baton/core/manager/` | Manager-mode PMO layer: post-processor around `create_plan()` producing project charter, scope map, team blueprint, role cards, knowledge plan, scope contracts, and context bundles. Config: `agent_baton/core/config/manager.py`. Models: `agent_baton/models/manager.py`. See `docs/internal/manager-mode-pmo-design.md`. | — | | `agent_baton/models/` | Pydantic data models — execution, beads, plans, decisions | [agent_baton/models/CLAUDE.md](agent_baton/models/CLAUDE.md) | | `agents/` | 30 distributable agent definitions (Markdown with frontmatter) | [agents/CLAUDE.md](agents/CLAUDE.md) | -| `references/` | 19 distributable reference procedures | [references/CLAUDE.md](references/CLAUDE.md) | +| `references/` | 20 distributable reference procedures | [references/CLAUDE.md](references/CLAUDE.md) | | `templates/` | `CLAUDE.md` + `settings.json` + skills installed to user projects | (do not modify `templates/CLAUDE.md` — it's a distributable artifact) | | `pmo-ui/` | React/Vite frontend served at `/pmo/` | [pmo-ui/CLAUDE.md](pmo-ui/CLAUDE.md) | | `tests/` | pytest suite (unit + integration) | [tests/CLAUDE.md](tests/CLAUDE.md) | @@ -142,13 +142,16 @@ cymbal impact # blast radius before edits | `BATON_BD_ENABLED` | Kept for backward compatibility. Has no effect after WP-G — `bd` is always required. | `1` | | `BATON_BD_BIN` | Path/name of the `bd` binary used by `BdClient`. | `bd` | | `BATON_BD_PREFIX` | Issue prefix passed to `bd init` so generated IDs match baton's `bd-` scheme. | `bd` | -| `BATON_PLANNER_HARD_GATE` | Enable hard validation gate that blocks structurally defective plans (deterministic checks — empty plans/phases, agent mismatches) | unset | +| `BATON_DEV_MODE` | Downgrade planner validation defects to warnings for local experimentation unless `BATON_PLANNER_HARD_GATE` is truthy. | unset | +| `BATON_PLANNER_WARN_ONLY` | Downgrade planner validation defects to warnings without enabling broader dev-mode behavior unless `BATON_PLANNER_HARD_GATE` is truthy. | unset | +| `BATON_PLANNER_HARD_GATE` | Force planner validation to block even when dev/warn-only mode is set. Blocking is already the default when neither warn-only flag is enabled. | unset | | `BATON_ARTIFACT_VALIDATION` | Derive extra gate commands from agent-created runnable artifacts (CI workflows, npm scripts, Playwright config, Makefile targets, pre-commit). Set to `0` to suppress derivation and run only the planned `gate.command`. | `1` | | `BATON_OTEL_ENABLED` | Enable OpenTelemetry JSONL export | unset | | `BATON_COMPLIANCE_FAIL_CLOSED` | Halt execution + raise on compliance audit write failure (regulated-domain). When unset/`0`, failures are logged + a bead warning is emitted, execution continues. Can be overridden per-plan via `MachinePlan.compliance_fail_closed` (plan value takes precedence). Also governs `baton comply-record` hook: `1` → exit 1 on write errors. | `0` | | `BATON_POLICY_FAIL_CLOSED` | Controls `baton policy-check` hook error handling: `0` → fail-open (bad stdin or unreadable policy → stderr warning, exit 0); `1` → exit 2 (blocks the tool call, shows stderr to the model). | `0` | | `BATON_GOAL_EVALUATOR` | Selects the goal evaluator strategy for `/goal` (G1): `stub` (deterministic, no LLM), `haiku` (Claude Haiku 4.5), or `opus` (Claude Opus 4.8). `haiku`/`opus` require `ANTHROPIC_API_KEY`; otherwise falls back to `stub`. | `haiku` | -| `BATON_TEAMS_BACKEND` | Selects the execution backend for team steps (A1). Two supported backends: `worktree` (default — parallel worktree-isolated dispatch, resumable via `baton execute resume`, nesting + full agent frontmatter honored) or `claude-teams` (native Agent Teams UX — inter-teammate messaging, shared task list, lead plan-approval; requires `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`; constraints: no resume, one team at a time, no nested teams, `skills`/`mcpServers` frontmatter not honored on teammates). Unknown values warn and fall back to `worktree`. | `worktree` | +| `BATON_TEAMS_BACKEND` | Selects the execution backend for team steps (A1). Two supported backends: `worktree` (default — parallel worktree-isolated dispatch, resumable via `baton execute resume`, nesting + full agent frontmatter honored) or `claude-teams` (native Agent Teams UX — inter-teammate messaging, shared task list, lead plan-approval; requires `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`; constraints: no resume, one team at a time, no nested teams, `skills`/`mcpServers` frontmatter not honored on teammates). Unknown values warn and fall back to `worktree` unless `BATON_TEAMS_BACKEND_STRICT=1`. | `worktree` | +| `BATON_TEAMS_BACKEND_STRICT` | When `1`, unknown `BATON_TEAMS_BACKEND` values raise `UnknownTeamBackendError` instead of silently falling back to `worktree`. | `0` | | `BATON_TEAMS_STRICT_RESUMABILITY` | When `1` AND `BATON_TEAMS_BACKEND=claude-teams` AND the plan has team phases the claude-teams backend cannot resume mid-flight under `long-running` budget, `baton plan`/`baton goal` refuses to save and exits 2. Default (`0`) downgrades the refusal to a warning. | `0` | | `BATON_PLAN_REVIEW` | Optional LLM plan-quality review after the deterministic pipeline: `off` (default) \| `haiku` \| `sonnet` \| `opus`. The deterministic pipeline has known limits in complexity assessment; default compensating controls are the structural hard gate and pre-flight human review in the spec queue — enable this for unattended/managed-mode planning. `sonnet` recommended. | off | | `BATON_MANAGER_ENRICH` | Optional LLM polish of the manager-mode project charter's `objective`/`background`/`assumptions` wording (never new scope/paths/facts): `off` (default) \| `haiku` \| `sonnet` \| `opus`. Runs post-deterministic-build — `ProjectCharterBuilder` itself stays fully deterministic; only `--manager-mode` plans call this. Requires `ANTHROPIC_API_KEY`; any failure (missing SDK/key, network, malformed response) silently falls back to the deterministic charter. | `off` | diff --git a/GEMINI.md b/GEMINI.md index 29017061..78861fb4 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -22,7 +22,7 @@ agent_baton/ Python package (the orchestration engine) bundles). Config: core/config/manager.py. Models: models/manager.py. See docs/internal/manager-mode-pmo-design.md. agents/ Distributable agent definitions (30 .md) -references/ Distributable reference procedures (19 .md) +references/ Distributable reference procedures (20 .md) templates/ CLAUDE.md + settings.json + skills/ — installed to targets scripts/ install.sh, install.ps1, record_spec_audit_beads.py tests/ pytest suite @@ -90,13 +90,16 @@ cymbal impact # blast radius before edits | `BATON_BD_ENABLED` | Kept for backward compatibility. Has no effect after WP-G — `bd` is always required. | `1` | | `BATON_BD_BIN` | Path/name of the `bd` binary used by `BdClient`. | `bd` | | `BATON_BD_PREFIX` | Issue prefix passed to `bd init` so generated IDs match baton's `bd-` scheme. | `bd` | -| `BATON_PLANNER_HARD_GATE` | Enable hard validation gate that blocks structurally defective plans (deterministic checks — empty plans/phases, agent mismatches) | unset | +| `BATON_DEV_MODE` | Downgrade planner validation defects to warnings for local experimentation unless `BATON_PLANNER_HARD_GATE` is truthy. | unset | +| `BATON_PLANNER_WARN_ONLY` | Downgrade planner validation defects to warnings without enabling broader dev-mode behavior unless `BATON_PLANNER_HARD_GATE` is truthy. | unset | +| `BATON_PLANNER_HARD_GATE` | Force planner validation to block even when dev/warn-only mode is set. Blocking is already the default when neither warn-only flag is enabled. | unset | | `BATON_ARTIFACT_VALIDATION` | Derive extra gate commands from agent-created runnable artifacts (CI workflows, npm scripts, Playwright config, Makefile targets, pre-commit). Set to `0` to suppress derivation and run only the planned `gate.command`. | `1` | | `BATON_OTEL_ENABLED` | Enable OpenTelemetry JSONL export | unset | | `BATON_COMPLIANCE_FAIL_CLOSED` | Halt execution + raise on compliance audit write failure (regulated-domain). When unset/`0`, failures are logged + a bead warning is emitted, execution continues. Can be overridden per-plan via `MachinePlan.compliance_fail_closed` (plan value takes precedence). Also governs `baton comply-record` hook: `1` → exit 1 on write errors. | `0` | | `BATON_POLICY_FAIL_CLOSED` | Controls `baton policy-check` hook error handling: `0` → fail-open (bad stdin or unreadable policy → stderr warning, exit 0); `1` → exit 2 (blocks the tool call, shows stderr to the model). | `0` | | `BATON_GOAL_EVALUATOR` | Selects the goal evaluator strategy for `/goal` (G1): `stub` (deterministic, no LLM), `haiku` (Claude Haiku 4.5), or `opus` (Claude Opus 4.8). `haiku`/`opus` require `ANTHROPIC_API_KEY`; otherwise falls back to `stub`. | `haiku` | -| `BATON_TEAMS_BACKEND` | Selects the execution backend for team steps (A1). Two supported backends: `worktree` (default — parallel worktree-isolated dispatch, resumable via `baton execute resume`, nesting + full agent frontmatter honored) or `claude-teams` (native Agent Teams UX — inter-teammate messaging, shared task list, lead plan-approval; requires `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`; constraints: no resume, one team at a time, no nested teams, `skills`/`mcpServers` frontmatter not honored on teammates). Unknown values warn and fall back to `worktree`. | `worktree` | +| `BATON_TEAMS_BACKEND` | Selects the execution backend for team steps (A1). Two supported backends: `worktree` (default — parallel worktree-isolated dispatch, resumable via `baton execute resume`, nesting + full agent frontmatter honored) or `claude-teams` (native Agent Teams UX — inter-teammate messaging, shared task list, lead plan-approval; requires `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`; constraints: no resume, one team at a time, no nested teams, `skills`/`mcpServers` frontmatter not honored on teammates). Unknown values warn and fall back to `worktree` unless `BATON_TEAMS_BACKEND_STRICT=1`. | `worktree` | +| `BATON_TEAMS_BACKEND_STRICT` | When `1`, unknown `BATON_TEAMS_BACKEND` values raise `UnknownTeamBackendError` instead of silently falling back to `worktree`. | `0` | | `BATON_TEAMS_STRICT_RESUMABILITY` | When `1` AND `BATON_TEAMS_BACKEND=claude-teams` AND the plan has team phases the claude-teams backend cannot resume mid-flight under `long-running` budget, `baton plan`/`baton goal` refuses to save and exits 2. Default (`0`) downgrades the refusal to a warning. | `0` | | `BATON_PLAN_REVIEW` | Optional LLM plan-quality review after the deterministic pipeline: `off` (default) \| `haiku` \| `sonnet` \| `opus`. The deterministic pipeline has known limits in complexity assessment; default compensating controls are the structural hard gate and pre-flight human review in the spec queue — enable this for unattended/managed-mode planning. `sonnet` recommended. | off | | `BATON_MANAGER_ENRICH` | Optional LLM polish of the manager-mode project charter's `objective`/`background`/`assumptions` wording (never new scope/paths/facts): `off` (default) \| `haiku` \| `sonnet` \| `opus`. Runs post-deterministic-build — `ProjectCharterBuilder` itself stays fully deterministic; only `--manager-mode` plans call this. Requires `ANTHROPIC_API_KEY`; any failure (missing SDK/key, network, malformed response) silently falls back to the deterministic charter. | `off` | diff --git a/Makefile b/Makefile index f89bf986..66d4a7f6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install test lint clean help +.PHONY: install test lint typecheck doctor ci-local clean help PYTHON ?= python3 VENV := .venv @@ -19,6 +19,25 @@ test: install ## Run tests test-verbose: install ## Run tests with verbose output $(VENV)/bin/python -m pytest tests/ -v --tb=short +lint: install ## Run lint if ruff is installed + @if $(VENV)/bin/python -c "import ruff" >/dev/null 2>&1; then \ + $(VENV)/bin/python -m ruff check .; \ + else \ + echo "ruff not installed; skipping lint"; \ + fi + +typecheck: install ## Run type checks if mypy is installed + @if $(VENV)/bin/python -c "import mypy" >/dev/null 2>&1; then \ + $(VENV)/bin/python -m mypy agent_baton; \ + else \ + echo "mypy not installed; skipping typecheck"; \ + fi + +doctor: install ## Run Baton doctor + $(VENV)/bin/python -m agent_baton.cli.main doctor + +ci-local: lint typecheck test doctor ## Run local checks available in this repo + clean: ## Remove build artifacts and venv rm -rf $(VENV) build/ dist/ *.egg-info .pytest_cache find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true diff --git a/README.md b/README.md index f8316e4b..61332751 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ between Claude and the engine — specifically `_print_action()` in ``` ┌────────────────────────────────────┐ │ ORCHESTRATOR │ - │ Reads 19 reference procedures │ + │ Reads 20 reference procedures │ └───────────────┬────────────────────┘ │ baton plan ─────┴───── baton execute @@ -158,7 +158,7 @@ scripts/install.sh # Linux/macOS The installer prompts for scope: user-level (`~/.claude/`) for all projects, or project-level (`.claude/`) for the current project only. It copies 30 agent -definitions, 19 reference procedures, a template `CLAUDE.md`, `settings.json` +definitions, 20 reference procedures, a template `CLAUDE.md`, `settings.json` hooks, and skills. It also attempts to install `bd` (the bead backend) via npm or Homebrew. @@ -291,7 +291,8 @@ not a hope. plans so more gaps surface. 5. **Assembly + validation** — assembles the final `MachinePlan` with gate commands, knowledge attachments, budget tier, and execution mode. The hard - gate (`BATON_PLANNER_HARD_GATE`) can block structurally defective plans. + gate blocks structurally defective plans by default; use + `BATON_PLANNER_WARN_ONLY` or `BATON_DEV_MODE` only for local experiments. ### Key commands @@ -874,7 +875,7 @@ on every invocation. Update scripts to use the new paths. ``` agents/ <- 30 agent definitions (Markdown + YAML frontmatter) -references/ <- 19 reference procedures (shared knowledge) +references/ <- 20 reference procedures (shared knowledge) templates/ <- CLAUDE.md, settings.json, skills, packs, and playbooks scripts/ <- Install scripts (Linux/macOS + Windows) and maintenance docs/ <- Architecture docs, ADRs, invariants, CLI/API reference, @@ -923,11 +924,14 @@ The variables a user is most likely to set. For the full internal list see | `BATON_GATE_RETRY` | Re-dispatch a failing step once with gate output appended; second failure is terminal | `0` | | `BATON_RUN_TOKEN_CEILING` | Per-run cumulative spend cap (USD). Survives `baton execute resume`. | unset | | `BATON_WORKTREE_STALE_HOURS` | Max age before the worktree GC reclaims a stale worktree | `4` | -| `BATON_PLANNER_HARD_GATE` | Block structurally defective plans (deterministic checks) | unset | +| `BATON_DEV_MODE` | Downgrade planner validation defects to warnings for local experimentation unless `BATON_PLANNER_HARD_GATE` is truthy | unset | +| `BATON_PLANNER_WARN_ONLY` | Downgrade planner validation defects to warnings without broader dev-mode behavior unless `BATON_PLANNER_HARD_GATE` is truthy | unset | +| `BATON_PLANNER_HARD_GATE` | Force planner validation to block even when dev/warn-only mode is set. Blocking is the default otherwise | unset | | `BATON_PLAN_REVIEW` | Optional LLM plan-quality review: `off` \| `haiku` \| `sonnet` \| `opus` | `off` | | `BATON_POLICY_FAIL_CLOSED` | `policy-check` hook: `0` fail-open, `1` blocks the tool call | `0` | | `BATON_COMPLIANCE_FAIL_CLOSED` | Halt execution on compliance audit write failure | `0` | -| `BATON_TEAMS_BACKEND` | Team-step backend: `worktree` (default, resumable) or `claude-teams` | `worktree` | +| `BATON_TEAMS_BACKEND` | Team-step backend: `worktree` (default, resumable) or `claude-teams`. Unknown values fall back unless strict mode is enabled | `worktree` | +| `BATON_TEAMS_BACKEND_STRICT` | Unknown `BATON_TEAMS_BACKEND` values raise `UnknownTeamBackendError` instead of falling back | `0` | | `BATON_SOULS_ENABLED` / `BATON_EXEC_BEADS_ENABLED` | Experimental feature flags (souls / executable beads) | unset | | `ANTHROPIC_API_KEY` | AI risk classification (`agent-baton[classify]`) and the planner classifier | none | @@ -999,7 +1003,7 @@ Requires Python 3.10+. Runtime dependencies: `pyyaml`, `pydantic`, `cryptography ## Project Status Agent Baton is in active development (v0.1.0). The orchestration engine, all 30 -agents, 19 reference procedures, knowledge delivery, bead memory system, PMO +agents, 20 reference procedures, knowledge delivery, bead memory system, PMO subsystem with end-to-end plan-to-merge workflow, REST API with webhooks, federated sync, event system, learning automation, and the improvement pipeline are implemented and tested. diff --git a/agent_baton/_bundled_agents/talent-builder.md b/agent_baton/_bundled_agents/talent-builder.md index 33d1a69a..762703cd 100644 --- a/agent_baton/_bundled_agents/talent-builder.md +++ b/agent_baton/_bundled_agents/talent-builder.md @@ -150,6 +150,49 @@ knowledge/[domain]/ **File:** `.claude/agents/[name].md` or `~/.claude/agents/[name].md` +**Generated-Agent Contract:** + +Every generated agent must follow this contract. Use +`references/agent-authoring.md` as the durable reference and +`.claude/templates/agents/*.md` as the starter source. + +Starter template files: +- `.claude/templates/agents/base-agent.md` +- `.claude/templates/agents/flavored-agent.md` +- `.claude/templates/agents/reviewer-agent.md` + +Required frontmatter fields: +- `name` +- `description` +- `model` +- `permissionMode` +- `tools` + +Recommended frontmatter fields: +- `owner` +- `status` +- `version` +- `created_by` +- `last_reviewed` +- `knowledge_packs` + +Required body sections: +- Mission +- Before Starting +- Knowledge References +- Principles +- Anti-Patterns +- Output Format + +Reference and tool rules: +- Avoid broad tools unless the agent mission requires them. Start read-only + (`Read`, `Glob`, `Grep`) for reviewers and researchers; add `Edit`, `Write`, + or `Bash` only when the agent must mutate files or run commands. +- Read back every generated file before reporting it as complete. +- Validate references exist before saving the agent. Every knowledge pack, + reference doc, skill, or template path named in the agent must resolve, or + the agent must explicitly state why it is optional. + **Template:** ```markdown @@ -163,14 +206,31 @@ permissionMode: [auto-edit for implementers, default for reviewers] color: [unused color] tools: [minimum needed — Read, Glob, Grep for read-only; add Write, Edit, Bash for implementers] +owner: [team or person responsible for maintenance] +status: draft +version: 0.1.0 +created_by: talent-builder +last_reviewed: [YYYY-MM-DD] +knowledge_packs: + - [knowledge/domain/overview.md or remove if none] --- # [Role Title] +## Mission + You are a [seniority + role]. [One-sentence mission.] ## Before Starting +1. Read this entire agent definition. +2. Read back every file listed under "Knowledge References"; do not rely on + stale memory. +3. Validate references exist and are relevant before using them. If a + reference is missing, report the gap. + +## Knowledge References + Read these knowledge packs before doing any work: - [path to knowledge pack files relevant to this agent] @@ -208,9 +268,19 @@ Return: **Agent quality checklist:** - [ ] Description is specific enough to trigger correctly -- [ ] Knowledge pack paths are referenced in "Before Starting" +- [ ] Required frontmatter fields exist: `name`, `description`, `model`, + `permissionMode`, `tools` +- [ ] Recommended frontmatter fields are filled when ownership is known: + `owner`, `status`, `version`, `created_by`, `last_reviewed`, + `knowledge_packs` +- [ ] Required body sections exist: Mission, Before Starting, Knowledge + References, Principles, Anti-Patterns, Output Format +- [ ] Knowledge pack paths are referenced in "Knowledge References" - [ ] Baked-in knowledge is concise (< 100 lines of domain content) -- [ ] Tools are minimum needed (principle of least privilege) +- [ ] Avoid broad tools; tools are minimum needed (principle of least + privilege) +- [ ] Read back the final agent file and validate references before reporting + completion - [ ] Output format matches the orchestrator's expectations - [ ] For flavored variants: references base role, same output format diff --git a/agent_baton/api/models/responses.py b/agent_baton/api/models/responses.py index d551e17f..9e963150 100644 --- a/agent_baton/api/models/responses.py +++ b/agent_baton/api/models/responses.py @@ -155,6 +155,10 @@ class PlanResponse(BaseModel): total_steps: int = Field(..., description="Total number of steps across all phases.") agents: list[str] = Field(default_factory=list, description="All agent names used in the plan.") pattern_source: Optional[str] = Field(default=None, description="Learned pattern that influenced this plan.") + plan_diagnostics: dict[str, Any] = Field( + default_factory=dict, + description="Concise planner diagnostics and knowledge-pack counters.", + ) created_at: str = Field(default="", description="ISO 8601 creation timestamp.") @classmethod @@ -172,6 +176,7 @@ def from_dataclass(cls, obj: object) -> PlanResponse: total_steps=obj.total_steps, # type: ignore[attr-defined] agents=obj.all_agents, # type: ignore[attr-defined] pattern_source=obj.pattern_source, # type: ignore[attr-defined] + plan_diagnostics=dict(getattr(obj, "plan_diagnostics", {})), created_at=obj.created_at, # type: ignore[attr-defined] ) diff --git a/agent_baton/api/planner_errors.py b/agent_baton/api/planner_errors.py new file mode 100644 index 00000000..0c5c88f8 --- /dev/null +++ b/agent_baton/api/planner_errors.py @@ -0,0 +1,29 @@ +"""Helpers for exposing planner validation failures through the API.""" +from __future__ import annotations + +from typing import Any + +from agent_baton.core.engine.planning.stages.validation import PlanQualityError + + +def plan_quality_error_detail(exc: PlanQualityError) -> dict[str, Any]: + """Build a structured HTTP error payload for a plan-quality rejection.""" + defects: list[dict[str, str]] = [] + for defect in getattr(exc, "defects", []) or []: + message = str(getattr(defect, "message", "") or "") + remediation = "" + marker = "Remediation:" + if marker in message: + remediation = message.split(marker, 1)[1].strip() + defects.append({ + "code": str(getattr(defect, "code", "") or ""), + "severity": str(getattr(defect, "severity", "") or ""), + "message": message, + "remediation": remediation, + }) + + return { + "error": "plan_quality_error", + "message": str(exc), + "defects": defects, + } diff --git a/agent_baton/api/routes/executions.py b/agent_baton/api/routes/executions.py index 52b6bab0..1ee2522d 100644 --- a/agent_baton/api/routes/executions.py +++ b/agent_baton/api/routes/executions.py @@ -9,6 +9,7 @@ """ from __future__ import annotations +import logging from typing import Any from fastapi import APIRouter, Depends, HTTPException @@ -22,10 +23,12 @@ ) from agent_baton.api.models.responses import ActionResponse, ExecutionResponse, RecordFeedbackResponse from agent_baton.core.engine.executor import ExecutionEngine +from agent_baton.core.engine.team_backends import UnknownTeamBackendError from agent_baton.core.runtime.decisions import DecisionManager from agent_baton.models.execution import MachinePlan router = APIRouter() +_log = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -87,14 +90,19 @@ async def start_execution( # Start the engine; returns the first action. try: first_action = engine.start(plan) + except UnknownTeamBackendError as exc: + _mark_execution_failed(engine, reason=str(exc)) + raise HTTPException(status_code=500, detail=str(exc)) from exc except RuntimeError as exc: + _mark_execution_failed(engine, reason=str(exc)) raise HTTPException(status_code=500, detail=str(exc)) from exc - # Gather all immediately dispatchable parallel actions. try: - parallel_actions = engine.next_actions() - except Exception: - parallel_actions = [first_action] + next_actions = _collect_next_actions(engine) + except HTTPException as exc: + if exc.status_code >= 500: + _mark_execution_failed(engine, reason=str(exc.detail)) + raise # Load state to build the response — start() saves it to disk. state = engine._load_state() # noqa: SLF001 @@ -104,11 +112,7 @@ async def start_execution( pending_count = _count_pending(decision_manager) execution = ExecutionResponse.from_dataclass(state, pending_decisions=pending_count) - next_actions = ( - [ActionResponse.from_dataclass(a) for a in parallel_actions] - if parallel_actions - else [ActionResponse.from_dataclass(first_action)] - ) + next_actions = next_actions or [ActionResponse.from_dataclass(first_action)] return { "execution": execution.model_dump(), @@ -439,12 +443,39 @@ def _assert_active_task(engine: ExecutionEngine, task_id: str) -> None: ) +def _mark_execution_failed(engine: ExecutionEngine, *, reason: str) -> None: + """Best-effort failure stamp for executions that persisted before startup failed.""" + try: + state = engine._load_state() # noqa: SLF001 + except Exception: + _log.warning("Could not load execution state after startup failure", exc_info=True) + return + if state is None: + return + try: + state.transition_to_failed(reason=reason) + engine._save_execution(state) # noqa: SLF001 + except Exception: + _log.warning( + "Could not persist failed execution state after startup failure", + exc_info=True, + ) + + def _collect_next_actions(engine: ExecutionEngine) -> list[ActionResponse]: """Return the next batch of dispatchable actions (parallel where possible). Attempts ``engine.next_actions()`` first for parallel dispatch. Falls back to ``engine.next_action()`` (single) if the parallel - method fails. Returns an empty list if both fail. + method fails. Returns an empty list if both fail, except for strict + team-backend configuration errors, which are surfaced to callers. + + Deliberate semantic split for ``UnknownTeamBackendError``: at start + time the caller marks the execution failed (nothing has run, the plan + can be re-POSTed), but mid-run the 500 leaves status ``running`` — the + error is a recoverable env misconfiguration and completed work must + stay resumable. Pinned by + ``test_unknown_backend_mid_run_keeps_execution_running``. Args: engine: The ``ExecutionEngine`` to query. @@ -459,11 +490,15 @@ def _collect_next_actions(engine: ExecutionEngine) -> list[ActionResponse]: parallel = engine.next_actions() if parallel: return [ActionResponse.from_dataclass(a) for a in parallel] + except UnknownTeamBackendError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc except Exception: _log.warning("next_actions() failed, falling back to next_action()", exc_info=True) try: single = engine.next_action() return [ActionResponse.from_dataclass(single)] + except UnknownTeamBackendError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc except Exception: _log.warning("next_action() failed, returning empty actions", exc_info=True) return [] diff --git a/agent_baton/api/routes/pmo.py b/agent_baton/api/routes/pmo.py index 2aedf18d..e66608e5 100644 --- a/agent_baton/api/routes/pmo.py +++ b/agent_baton/api/routes/pmo.py @@ -33,7 +33,9 @@ from sse_starlette.sse import EventSourceResponse from agent_baton.api.deps import get_bus, get_central_store, get_forge_session, get_pmo_scanner, get_pmo_store +from agent_baton.api.planner_errors import plan_quality_error_detail from agent_baton.core.events.bus import EventBus +from agent_baton.core.engine.planning.stages.validation import PlanQualityError from agent_baton.api.models.requests import ( ApproveForgeRequest, BatchResolveRequest, @@ -786,6 +788,17 @@ async def forge_plan( ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc + except PlanQualityError as exc: + _publish_forge_progress( + queue, + "failed", + 100, + f"Plan validation failed: {exc}", + ) + raise HTTPException( + status_code=422, + detail=plan_quality_error_detail(exc), + ) from exc except Exception as exc: raise HTTPException( status_code=500, @@ -953,6 +966,11 @@ async def forge_regenerate( task_type=req.task_type, priority=req.priority, ) + except PlanQualityError as exc: + raise HTTPException( + status_code=422, + detail=plan_quality_error_detail(exc), + ) from exc except Exception as exc: raise HTTPException(status_code=500, detail=f"Regeneration failed: {exc}") from exc diff --git a/agent_baton/api/routes/spec_queue.py b/agent_baton/api/routes/spec_queue.py index 2a32e35a..7ca9c2c3 100644 --- a/agent_baton/api/routes/spec_queue.py +++ b/agent_baton/api/routes/spec_queue.py @@ -30,6 +30,8 @@ FireSpecDraftResponse, SpecDraftResponse, ) +from agent_baton.api.planner_errors import plan_quality_error_detail +from agent_baton.core.engine.planning.stages.validation import PlanQualityError from agent_baton.core.federate.spec_draft_store import SpecDraftStore from agent_baton.models.spec_draft import ReviewData @@ -352,6 +354,11 @@ async def fire_spec_draft( ) except HTTPException: raise + except PlanQualityError as exc: + raise HTTPException( + status_code=422, + detail=plan_quality_error_detail(exc), + ) from exc except Exception as exc: logger.error("fire_spec_draft: plan generation failed for %s: %s", spec_id, exc) raise HTTPException( diff --git a/agent_baton/cli/commands/diagnostics_cmd.py b/agent_baton/cli/commands/diagnostics_cmd.py new file mode 100644 index 00000000..ed86ee39 --- /dev/null +++ b/agent_baton/cli/commands/diagnostics_cmd.py @@ -0,0 +1,1134 @@ +"""``baton doctor`` -- developer-facing installation health report.""" +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from importlib import metadata +from pathlib import Path +from typing import Any + + +_PYTHON_MIN = (3, 10) + + +@dataclass(frozen=True) +class DoctorCheck: + """Single doctor check result.""" + + id: str + label: str + status: str + message: str + details: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "label": self.label, + "status": self.status, + "message": self.message, + "details": self.details, + } + + +def register(subparsers: argparse._SubParsersAction) -> argparse.ArgumentParser: # type: ignore[type-arg] + parser = subparsers.add_parser( + "doctor", + help=( + "Check Baton installation health, knowledge packs, assurance packs, " + "PMO UI assets, and optional local CLIs" + ), + description=( + "Check Baton installation health, knowledge packs, assurance packs, " + "PMO UI assets, and optional local CLIs." + ), + ) + parser.add_argument( + "--json", + action="store_true", + help="Emit the doctor report as JSON", + ) + return parser + + +def handler(args: argparse.Namespace) -> None: + payload = build_report(project_root=Path.cwd()) + if getattr(args, "json", False): + print(json.dumps(payload, indent=2)) + else: + print(render_report(payload)) + if not payload["ok"]: + raise SystemExit(1) + + +def build_report(project_root: Path | None = None) -> dict[str, Any]: + root = (project_root or Path.cwd()).expanduser().resolve() + checks = [ + _check_python(), + _check_package_version(), + _check_bundled_agents(), + _check_project_agents(root), + _check_knowledge_packs(root), + _check_assurance_packs(root), + _check_pmo_ui_assets(root), + _check_package_resources(), + _check_bd(), + _check_beads_workspace(root), + _check_git(root), + _check_git_worktree(root), + _check_claude_cli(), + _check_team_context(root), + _check_planner_validation(root), + _check_terminology(), + ] + summary = _summary(checks) + return { + "schema_version": 1, + "ok": summary["error"] == 0, + "project_root": str(root), + "summary": summary, + "checks": [check.to_dict() for check in checks], + } + + +def render_report(payload: dict[str, Any]) -> str: + summary = payload["summary"] + lines = [ + "Baton doctor", + f"Project root: {payload['project_root']}", + ( + "Summary: " + f"ok={summary['ok']} warnings={summary['warning']} " + f"errors={summary['error']}" + ), + "", + ] + for check in payload["checks"]: + status = str(check["status"]).upper() + label = check["label"] + message = check["message"] + lines.append(f"[{status}] {label}: {message}") + return "\n".join(lines) + + +def _summary(checks: list[DoctorCheck]) -> dict[str, int]: + counts = {"ok": 0, "warning": 0, "error": 0} + for check in checks: + counts[check.status] = counts.get(check.status, 0) + 1 + return counts + + +def _check_python() -> DoctorCheck: + current = sys.version_info + required = f">={_PYTHON_MIN[0]}.{_PYTHON_MIN[1]}" + version = f"{current.major}.{current.minor}.{current.micro}" + if (current.major, current.minor) < _PYTHON_MIN: + return DoctorCheck( + id="python", + label="Python version", + status="error", + message=f"{version} is below Baton requirement {required}", + details={"version": version, "requires": required}, + ) + return DoctorCheck( + id="python", + label="Python version", + status="ok", + message=f"{version} satisfies Baton requirement {required}", + details={"version": version, "requires": required}, + ) + + +def _check_package_version() -> DoctorCheck: + version = _package_version() + return DoctorCheck( + id="package_version", + label="Package version", + status="ok", + message=f"agent-baton {version}", + details={"distribution": "agent-baton", "version": version}, + ) + + +def _package_version() -> str: + try: + return metadata.version("agent-baton") + except metadata.PackageNotFoundError: + try: + import agent_baton + + return getattr(agent_baton, "__version__", "dev") + except Exception: + return "dev" + + +def _check_bundled_agents() -> DoctorCheck: + names = _bundled_agent_names() + if not names: + return DoctorCheck( + id="bundled_agents", + label="Bundled agents", + status="error", + message="No bundled agents were found in package resources", + details={"count": 0, "names": []}, + ) + status = "ok" if "talent-builder" in names else "warning" + message = ( + f"{len(names)} bundled agents available; talent-builder present" + if status == "ok" + else f"{len(names)} bundled agents available; talent-builder missing" + ) + return DoctorCheck( + id="bundled_agents", + label="Bundled agents", + status=status, + message=message, + details={"count": len(names), "names": names}, + ) + + +def _bundled_agent_names() -> list[str]: + try: + import importlib.resources as pkg_resources + + root = pkg_resources.files("agent_baton").joinpath("_bundled_agents") + if not root.is_dir(): # type: ignore[union-attr] + return [] + names = [] + for entry in root.iterdir(): # type: ignore[union-attr] + name = getattr(entry, "name", "") + if name.endswith(".md") and name != "CLAUDE.md": + names.append(Path(name).stem) + return sorted(names) + except Exception: + return [] + + +def _check_project_agents(project_root: Path) -> DoctorCheck: + agents_dir = project_root / ".claude" / "agents" + names = _markdown_stems(agents_dir) + validation = _validate_agent_dir(agents_dir) + if not names: + return DoctorCheck( + id="project_agents", + label="Project agents", + status="warning", + message=f"No project agents found at {agents_dir}", + details={ + "path": str(agents_dir), + "count": 0, + "names": [], + **validation, + }, + ) + validation_errors = validation.get("validation_errors", 0) + validation_error = validation.get("validation_error") + status = "warning" if validation_errors or validation_error else "ok" + suffix = ( + f"; {validation_errors} validation errors" + if validation_errors + else f"; validation unavailable: {validation_error}" + if validation_error + else "" + ) + return DoctorCheck( + id="project_agents", + label="Project agents", + status=status, + message=f"{len(names)} project agents found{suffix}", + details={ + "path": str(agents_dir), + "count": len(names), + "names": names, + **validation, + }, + ) + + +def _check_knowledge_packs(project_root: Path) -> DoctorCheck: + project_dir = project_root / ".claude" / "knowledge" + global_dir = Path.home() / ".claude" / "knowledge" + project = _count_pack_dirs(project_dir, manifest_name="knowledge.yaml") + global_ = _count_pack_dirs(global_dir, manifest_name="knowledge.yaml") + registry_details = _load_knowledge_registry_details(project_root) + total = project["count"] + global_["count"] + missing_manifest_count = ( + (project["count"] - project["with_manifest"]) + + (global_["count"] - global_["with_manifest"]) + ) + registry_degraded_count = registry_details.get("registry_degraded_count", 0) + registry_error = registry_details.get("registry_error") + if total == 0: + return DoctorCheck( + id="knowledge_packs", + label="Knowledge packs", + status="warning", + message=( + "No knowledge packs found; expected manifests are named " + "knowledge.yaml" + ), + details={ + **_pack_details(project_dir, global_dir, project, global_), + **registry_details, + }, + ) + degraded = bool( + missing_manifest_count or registry_degraded_count or registry_error + ) + status = "warning" if degraded else "ok" + message_bits = [f"{total} knowledge packs found"] + if missing_manifest_count: + message_bits.append( + f"{missing_manifest_count} missing knowledge.yaml" + ) + else: + message_bits.append( + f"{project['with_manifest'] + global_['with_manifest']} have knowledge.yaml" + ) + if registry_degraded_count: + message_bits.append( + f"{registry_degraded_count} degraded in registry" + ) + if registry_error: + message_bits.append("registry diagnostics unavailable") + return DoctorCheck( + id="knowledge_packs", + label="Knowledge packs", + status=status, + message="; ".join(message_bits), + details={ + **_pack_details(project_dir, global_dir, project, global_), + **registry_details, + }, + ) + + +def _check_assurance_packs(project_root: Path) -> DoctorCheck: + project_dir = project_root / ".claude" / "packs" + global_dir = Path.home() / ".claude" / "packs" + project = _count_pack_dirs(project_dir, manifest_name="pack.json") + global_ = _count_pack_dirs(global_dir, manifest_name="pack.json") + validation = _validate_assurance_pack_dirs(project_dir, global_dir) + total = project["count"] + global_["count"] + invalid_count = validation.get("invalid_count", 0) + validation_error = validation.get("validation_error") + if total == 0: + return DoctorCheck( + id="assurance_packs", + label="Assurance packs", + status="warning", + message="No assurance packs found at .claude/packs", + details={ + **_pack_details(project_dir, global_dir, project, global_), + **validation, + }, + ) + status = "warning" if invalid_count or validation_error else "ok" + suffix = ( + f"; {invalid_count} invalid" + if invalid_count + else f"; validation unavailable: {validation_error}" + if validation_error + else "" + ) + return DoctorCheck( + id="assurance_packs", + label="Assurance packs", + status=status, + message=f"{total} assurance packs found{suffix}", + details={ + **_pack_details(project_dir, global_dir, project, global_), + **validation, + }, + ) + + +def _check_pmo_ui_assets(project_root: Path) -> DoctorCheck: + pmo_root = project_root / "pmo-ui" + dist_index = pmo_root / "dist" / "index.html" + source_index = pmo_root / "index.html" + source_app = pmo_root / "src" / "App.tsx" + dist_exists = dist_index.is_file() + source_exists = source_index.is_file() and source_app.is_file() + if dist_exists: + status = "ok" + message = "Built PMO UI static assets are available" + elif source_exists: + status = "warning" + message = "PMO UI source exists, but pmo-ui/dist/index.html is not built" + else: + status = "warning" + message = "PMO UI assets were not found" + return DoctorCheck( + id="pmo_ui_assets", + label="PMO UI assets", + status=status, + message=message, + details={ + "pmo_root": str(pmo_root), + "dist_index": str(dist_index), + "dist_exists": dist_exists, + "source_exists": source_exists, + }, + ) + + +def _check_package_resources() -> DoctorCheck: + resources = { + "bundled_agents": _package_resource_state("_bundled_agents", "*.md"), + "references": _package_resource_state("_bundled_references", "*.md"), + "templates": _package_resource_state("_bundled_templates", "*"), + "pmo_static_assets": _package_resource_state("_bundled_pmo_ui", "*"), + } + missing = [ + name + for name, state in resources.items() + if state["status"] != "ok" + ] + if missing: + return DoctorCheck( + id="package_resources", + label="Package resources", + status="warning", + message=( + "Package-resource audit found missing optional resources: " + + ", ".join(missing) + ), + details={"resources": resources}, + ) + return DoctorCheck( + id="package_resources", + label="Package resources", + status="ok", + message="Package-resource audit found all expected resource groups", + details={"resources": resources}, + ) + + +def _package_resource_state(resource_dir: str, pattern: str) -> dict[str, Any]: + try: + import importlib.resources as pkg_resources + + root = pkg_resources.files("agent_baton").joinpath(resource_dir) + if not root.is_dir(): # type: ignore[union-attr] + return { + "status": "warning", + "count": 0, + "path": f"agent_baton/{resource_dir}", + "message": "resource directory is not bundled", + } + entries = [ + entry + for entry in root.iterdir() # type: ignore[union-attr] + if _resource_entry_matches(getattr(entry, "name", ""), pattern) + ] + if not entries: + return { + "status": "warning", + "count": 0, + "path": f"agent_baton/{resource_dir}", + "message": "resource directory is bundled but empty", + } + return { + "status": "ok", + "count": len(entries), + "path": f"agent_baton/{resource_dir}", + "message": "resource directory is bundled", + } + except Exception as exc: + return { + "status": "warning", + "count": 0, + "path": f"agent_baton/{resource_dir}", + "message": f"resource audit skipped: {exc}", + } + + +def _resource_entry_matches(name: str, pattern: str) -> bool: + if pattern == "*": + return name not in {"", "__pycache__"} + if pattern == "*.md": + return name.endswith(".md") + return bool(name) + + +def _check_bd() -> DoctorCheck: + return _check_optional_cli( + check_id="bd", + label="bd availability", + executable="bd", + missing_message="Optional bd CLI not found on PATH", + ) + + +def _check_beads_workspace(project_root: Path) -> DoctorCheck: + beads_dir = project_root / ".beads" + expected_files = [ + "config.yaml", + "interactions.jsonl", + "metadata.json", + ] + present_files = [ + name for name in expected_files if (beads_dir / name).is_file() + ] + missing_files = [ + name for name in expected_files if name not in present_files + ] + exists = beads_dir.is_dir() + status = "ok" if exists and not missing_files else "warning" + if status == "ok": + message = "Beads workspace files are present" + elif not exists: + message = f"Beads workspace directory is missing at {beads_dir}" + else: + message = ( + "Beads workspace is missing expected files: " + + ", ".join(missing_files) + ) + return DoctorCheck( + id="beads_workspace", + label="Beads workspace", + status=status, + message=message, + details={ + "path": str(beads_dir), + "exists": exists, + "missing_files": missing_files, + "present_files": present_files, + }, + ) + + +def _check_claude_cli() -> DoctorCheck: + return _check_optional_cli( + check_id="claude_cli", + label="Claude CLI availability", + executable="claude", + missing_message="Optional Claude CLI not found on PATH", + ) + + +def _check_optional_cli( + *, + check_id: str, + label: str, + executable: str, + missing_message: str, +) -> DoctorCheck: + found = shutil.which(executable) + if not found: + return DoctorCheck( + id=check_id, + label=label, + status="warning", + message=missing_message, + details={"executable": executable, "path": None}, + ) + return DoctorCheck( + id=check_id, + label=label, + status="ok", + message=f"{executable} found at {found}", + details={"executable": executable, "path": found}, + ) + + +def _check_git(project_root: Path) -> DoctorCheck: + if not shutil.which("git"): + return DoctorCheck( + id="git", + label="Git repo status", + status="warning", + message="git executable not found on PATH", + details={"path": None}, + ) + + inside = _git(["rev-parse", "--is-inside-work-tree"], project_root) + if inside["returncode"] != 0 or inside["stdout"].strip() != "true": + return DoctorCheck( + id="git", + label="Git repo status", + status="warning", + message="Project root is not inside a git work tree", + details=inside, + ) + + branch = _git(["branch", "--show-current"], project_root) + status = _git(["status", "--porcelain"], project_root) + dirty_lines = [ + line for line in status["stdout"].splitlines() if line.strip() + ] + if status["returncode"] != 0: + return DoctorCheck( + id="git", + label="Git repo status", + status="warning", + message="Unable to read git status", + details={"status": status, "branch": branch["stdout"].strip()}, + ) + if dirty_lines: + return DoctorCheck( + id="git", + label="Git repo status", + status="warning", + message=f"Git work tree has {len(dirty_lines)} changed paths", + details={ + "branch": branch["stdout"].strip(), + "dirty_count": len(dirty_lines), + "dirty_paths": dirty_lines[:20], + }, + ) + return DoctorCheck( + id="git", + label="Git repo status", + status="ok", + message="Git work tree is clean", + details={"branch": branch["stdout"].strip(), "dirty_count": 0}, + ) + + +def _check_git_worktree(project_root: Path) -> DoctorCheck: + if not shutil.which("git"): + return DoctorCheck( + id="git_worktree", + label="Git worktree topology", + status="warning", + message="git executable not found on PATH", + details={ + "branch": None, + "git_dir": None, + "git_common_dir": None, + "is_linked_worktree": False, + "is_submodule": False, + "detached_head": None, + "path": None, + }, + ) + + inside = _git(["rev-parse", "--is-inside-work-tree"], project_root) + if inside["returncode"] != 0 or inside["stdout"].strip() != "true": + return DoctorCheck( + id="git_worktree", + label="Git worktree topology", + status="warning", + message="Git metadata is not readable for this project root", + details={ + "branch": None, + "git_dir": None, + "git_common_dir": None, + "is_linked_worktree": False, + "is_submodule": False, + "detached_head": None, + "inside_work_tree": inside["stdout"].strip(), + "inside_work_tree_probe": inside, + }, + ) + + git_dir_probe = _git(["rev-parse", "--git-dir"], project_root) + git_common_dir_probe = _git(["rev-parse", "--git-common-dir"], project_root) + superproject_probe = _git( + ["rev-parse", "--show-superproject-working-tree"], + project_root, + ) + branch_probe = _git(["branch", "--show-current"], project_root) + head_probe = _git(["rev-parse", "--abbrev-ref", "HEAD"], project_root) + + probes = { + "git_dir": git_dir_probe, + "git_common_dir": git_common_dir_probe, + "superproject": superproject_probe, + "branch": branch_probe, + "head": head_probe, + } + unreadable = [ + name for name, probe in probes.items() if probe["returncode"] != 0 + ] + branch = branch_probe["stdout"].strip() or None + git_dir = git_dir_probe["stdout"].strip() or None + git_common_dir = git_common_dir_probe["stdout"].strip() or None + head_name = head_probe["stdout"].strip() + is_submodule = bool(superproject_probe["stdout"].strip()) + is_linked_worktree = bool( + git_dir + and git_common_dir + and git_dir != git_common_dir + and not is_submodule + ) + detached_head = head_name == "HEAD" + details = { + "branch": branch, + "git_dir": git_dir, + "git_common_dir": git_common_dir, + "is_linked_worktree": is_linked_worktree, + "is_submodule": is_submodule, + "detached_head": detached_head, + } + if unreadable: + details["probe_failures"] = unreadable + details["probes"] = probes + return DoctorCheck( + id="git_worktree", + label="Git worktree topology", + status="warning", + message=( + "Git metadata could not be fully read: " + + ", ".join(unreadable) + ), + details=details, + ) + return DoctorCheck( + id="git_worktree", + label="Git worktree topology", + status="ok", + message="Git worktree metadata is readable", + details=details, + ) + + +def _git(args: list[str], cwd: Path) -> dict[str, Any]: + try: + proc = subprocess.run( + ["git", "-C", str(cwd), "--no-optional-locks", *args], + capture_output=True, + text=True, + timeout=3, + check=False, + ) + except Exception as exc: + return {"returncode": 1, "stdout": "", "stderr": str(exc)} + return { + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + + +# os.access(W_OK) reads POSIX permission bits only; on Windows it ignores +# NTFS ACLs, so a directory can pass this check yet still refuse writes. +_WRITABLE_CHECK_CAVEAT = ( + "metadata check only; NTFS ACLs are not evaluated, so writes may still " + "be denied at runtime" +) + + +def _check_team_context(project_root: Path) -> DoctorCheck: + path = project_root / ".claude" / "team-context" + if not path.is_dir(): + return DoctorCheck( + id="team_context", + label=".claude/team-context", + status="warning", + message=f"{path} does not exist", + details={ + "path": str(path), + "writable": False, + "writable_check": "metadata-only", + "writable_check_caveat": _WRITABLE_CHECK_CAVEAT, + }, + ) + writable, error = _probe_writable_directory(path) + if not writable: + return DoctorCheck( + id="team_context", + label=".claude/team-context", + status="warning", + message=f"{path} does not appear writable: {error}", + details={ + "path": str(path), + "writable": False, + "writable_check": "metadata-only", + "writable_check_caveat": _WRITABLE_CHECK_CAVEAT, + }, + ) + return DoctorCheck( + id="team_context", + label=".claude/team-context", + status="ok", + message=f"{path} appears writable", + details={ + "path": str(path), + "writable": True, + "writable_check": "metadata-only", + "writable_check_caveat": _WRITABLE_CHECK_CAVEAT, + }, + ) + + +def _probe_writable_directory(path: Path) -> tuple[bool, str]: + try: + if not path.exists(): + return False, "path does not exist" + if not path.is_dir(): + return False, "path is not a directory" + if not os.access(path, os.W_OK): + return False, "metadata check denied write access" + except OSError as exc: + return False, str(exc) + return True, "" + + +def _check_terminology() -> DoctorCheck: + return DoctorCheck( + id="terminology", + label="Terminology", + status="ok", + message=( + "Canonical agent is talent-builder; knowledge pack manifests use " + "knowledge.yaml; assurance packs live under .claude/packs" + ), + details={ + "canonical_agent": "talent-builder", + "knowledge_manifest": "knowledge.yaml", + "knowledge_pack_dir": ".claude/knowledge", + "assurance_pack_dir": ".claude/packs", + }, + ) + + +def _check_planner_validation(project_root: Path) -> DoctorCheck: + plan_candidates = _saved_plan_candidates(project_root) + plan_path, active_task_state = _select_saved_plan_for_validation(project_root) + details: dict[str, Any] = { + "active_task_id": active_task_state["active_task_id"], + "active_task_source": active_task_state["active_task_source"], + "plan_candidates": [str(path) for path in plan_candidates], + "plan_path": None, + "machine_plan_importable": False, + "validator_importable": False, + "findings_count": 0, + "error_count": 0, + "warning_count": 0, + } + if "active_task_sqlite_probe" in active_task_state: + details["active_task_sqlite_probe"] = active_task_state[ + "active_task_sqlite_probe" + ] + try: + from agent_baton.models.execution import MachinePlan + + details["machine_plan_importable"] = MachinePlan.__name__ == "MachinePlan" + except Exception as exc: + details["import_error"] = str(exc) + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="error", + message="MachinePlan import failed", + details=details, + ) + + try: + from agent_baton.cli.commands.execution.plan_validate_cmd import ( + _validate_plan, + ) + + details["validator_importable"] = True + except Exception as exc: + details["import_error"] = str(exc) + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="error", + message="Plan validation import failed", + details=details, + ) + + if plan_path is None: + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="warning", + message="No saved plan is available to validate", + details=details, + ) + + details["plan_path"] = str(plan_path) + if active_task_state["active_plan_missing"]: + details["active_plan_missing"] = True + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="warning", + message=f"Active task plan is missing: {plan_path}", + details=details, + ) + try: + data = json.loads(plan_path.read_text(encoding="utf-8")) + except Exception as exc: + details["read_error"] = str(exc) + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="error", + message=f"Saved plan could not be parsed: {exc}", + details=details, + ) + + if not isinstance(data, dict): + details["validation_error"] = ( + "Saved plan JSON must be an object at the top level" + ) + details["plan_data_type"] = type(data).__name__ + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="error", + message="Saved plan JSON has invalid top-level shape", + details=details, + ) + + try: + findings = _validate_plan(data) + except Exception as exc: + details["validation_error"] = str(exc) + details["validation_exception_type"] = type(exc).__name__ + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="error", + message=f"Saved plan validation could not run: {exc}", + details=details, + ) + + error_count = sum( + 1 for finding in findings if finding.get("severity") == "error" + ) + warning_count = sum( + 1 for finding in findings if finding.get("severity") == "warning" + ) + details.update({ + "findings_count": len(findings), + "error_count": error_count, + "warning_count": warning_count, + "findings": findings, + }) + if error_count: + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="error", + message=( + f"Saved plan validation found {error_count} errors and " + f"{warning_count} warnings" + ), + details=details, + ) + if warning_count: + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="warning", + message=f"Saved plan validation found {warning_count} warnings", + details=details, + ) + return DoctorCheck( + id="planner_validation", + label="Planner validation", + status="ok", + message="Saved plan validation passed with no findings", + details=details, + ) + + +def _markdown_stems(path: Path) -> list[str]: + if not path.is_dir(): + return [] + return sorted(p.stem for p in path.glob("*.md") if p.is_file()) + + +def _validate_agent_dir(path: Path) -> dict[str, Any]: + if not path.is_dir(): + return { + "validated_count": 0, + "validation_warnings": 0, + "validation_errors": 0, + } + try: + from agent_baton.core.govern.validator import AgentValidator + + results = AgentValidator().validate_directory(path) + except Exception as exc: + return { + "validated_count": 0, + "validation_warnings": 0, + "validation_errors": 0, + "validation_error": str(exc), + } + return { + "validated_count": len(results), + "validation_warnings": sum(1 for result in results if result.warnings), + "validation_errors": sum(1 for result in results if result.errors), + } + + +def _count_pack_dirs(path: Path, *, manifest_name: str) -> dict[str, Any]: + if not path.is_dir(): + return {"count": 0, "with_manifest": 0, "names": []} + dirs = sorted(p for p in path.iterdir() if p.is_dir()) + return { + "count": len(dirs), + "with_manifest": sum(1 for p in dirs if (p / manifest_name).is_file()), + "names": [p.name for p in dirs], + } + + +def _pack_details( + project_dir: Path, + global_dir: Path, + project: dict[str, Any], + global_: dict[str, Any], +) -> dict[str, Any]: + return { + "project_path": str(project_dir), + "global_path": str(global_dir), + "project_count": project["count"], + "global_count": global_["count"], + "project_with_manifest": project["with_manifest"], + "global_with_manifest": global_["with_manifest"], + "project_names": project["names"], + "global_names": global_["names"], + } + + +def _load_knowledge_registry_details(project_root: Path) -> dict[str, Any]: + try: + from agent_baton.core.orchestration.knowledge_registry import ( + KnowledgeRegistry, + ) + + registry = KnowledgeRegistry() + loaded = registry.load_default_paths(project_root=project_root) + return { + "registry_loaded_count": loaded, + "registry_well_formed_count": registry.well_formed_pack_count, + "registry_degraded_count": registry.degraded_pack_count, + "registry_degraded_names": sorted(registry.degraded_pack_names), + } + except Exception as exc: + return { + "registry_loaded_count": 0, + "registry_well_formed_count": 0, + "registry_degraded_count": 0, + "registry_degraded_names": [], + "registry_error": str(exc), + } + + +def _validate_assurance_pack_dirs(*roots: Path) -> dict[str, Any]: + try: + from agent_baton.core.govern.packs import validate_pack + except Exception as exc: + return {"validation_error": str(exc), "invalid_count": 0} + + invalid: list[dict[str, Any]] = [] + for root in roots: + if not root.is_dir(): + continue + for pack_dir in sorted(p for p in root.iterdir() if p.is_dir()): + errors = validate_pack(pack_dir) + if errors: + invalid.append({ + "pack": pack_dir.name, + "path": str(pack_dir), + "errors": [str(error) for error in errors], + }) + return {"invalid_count": len(invalid), "invalid_packs": invalid} + + +def _find_saved_plan(project_root: Path) -> Path | None: + for candidate in _saved_plan_candidates(project_root): + if candidate.is_file(): + return candidate + return None + + +def _select_saved_plan_for_validation( + project_root: Path, +) -> tuple[Path | None, dict[str, Any]]: + context_root = project_root / ".claude" / "team-context" + ( + active_task_id, + active_task_source, + active_task_details, + ) = _resolve_active_task_for_validation( + context_root + ) + active_task_state: dict[str, Any] = { + "active_task_id": active_task_id, + "active_task_source": active_task_source, + "active_plan_missing": False, + **active_task_details, + } + if active_task_id: + active_plan_path = ( + context_root / "executions" / active_task_id / "plan.json" + ) + if active_plan_path.is_file(): + return active_plan_path, active_task_state + active_task_state["active_plan_missing"] = True + return active_plan_path, active_task_state + return _find_saved_plan(project_root), active_task_state + + +def _resolve_active_task_for_validation( + context_root: Path, +) -> tuple[str | None, str | None, dict[str, Any]]: + active_task_id = os.environ.get("BATON_TASK_ID", "").strip() + if active_task_id: + return active_task_id, "env", {} + + active_task_id, active_task_details = _read_active_task_id_from_sqlite( + context_root + ) + if active_task_id: + return active_task_id, "sqlite", active_task_details + + active_task_id = _read_active_task_id_from_file_marker(context_root) + if active_task_id: + return active_task_id, "file", active_task_details + return None, None, active_task_details + + +def _read_active_task_id_from_sqlite( + context_root: Path, +) -> tuple[str | None, dict[str, Any]]: + db_path = context_root / "baton.db" + from agent_baton.core.storage.active_task import ( + read_active_task_id_from_db_copy, + ) + + probe = read_active_task_id_from_db_copy(db_path) + if probe.degraded: + return ( + None, + {"active_task_sqlite_probe": probe.degradation_details()}, + ) + return probe.task_id, {} + + +def _read_active_task_id_from_file_marker(context_root: Path) -> str | None: + try: + from agent_baton.core.engine.persistence import StatePersistence + + return StatePersistence.get_active_task_id(context_root) + except Exception: + return None + +def _saved_plan_candidates(project_root: Path) -> list[Path]: + context_root = project_root / ".claude" / "team-context" + candidates = [ + context_root / "plan.json", + project_root / "plan.json", + ] + executions_dir = context_root / "executions" + if executions_dir.is_dir(): + for task_dir in sorted( + child for child in executions_dir.iterdir() if child.is_dir() + ): + candidates.append(task_dir / "plan.json") + return candidates diff --git a/agent_baton/cli/commands/distribute/install.py b/agent_baton/cli/commands/distribute/install.py index 81e8b656..55ccd35a 100644 --- a/agent_baton/cli/commands/distribute/install.py +++ b/agent_baton/cli/commands/distribute/install.py @@ -97,6 +97,12 @@ def _verify_install(base: Path, agents_dir: Path, refs_dir: Path, team_ctx: Path if not ref_files: issues.append("No reference .md files found in " + str(refs_dir)) + # Check generated-agent starter templates + template_agent_dir = base / "templates" / "agents" + template_agent_files = list(template_agent_dir.glob("*.md")) + if not template_agent_files: + issues.append("No starter template .md files found in " + str(template_agent_dir)) + # Check team-context is writable try: test_file = team_ctx / ".verify-test" @@ -207,6 +213,7 @@ def _cmd_install(args: argparse.Namespace) -> None: agents_src = source / "agents" refs_src = source / "references" + agent_templates_src = source / "templates" / "agents" claude_md_src = source / "templates" / "CLAUDE.md" settings_src = source / "templates" / "settings.json" @@ -231,8 +238,16 @@ def _cmd_install(args: argparse.Namespace) -> None: team_ctx = base / "team-context" knowledge_dir = base / "knowledge" skills_dir = base / "skills" - - for d in (agent_target, ref_target, team_ctx, knowledge_dir, skills_dir): + template_agent_target = base / "templates" / "agents" + + for d in ( + agent_target, + ref_target, + team_ctx, + knowledge_dir, + skills_dir, + template_agent_target, + ): d.mkdir(parents=True, exist_ok=True) # Agents + references: always overwrite on upgrade (these improve between versions) @@ -252,6 +267,13 @@ def _cmd_install(args: argparse.Namespace) -> None: if _copy_file(src_file, dst_file, force=ref_force): ref_count += 1 + template_agent_count = 0 + if agent_templates_src.is_dir(): + for src_file in sorted(agent_templates_src.glob("*.md")): + dst_file = template_agent_target / src_file.name + if _copy_file(src_file, dst_file, force=force or upgrade): + template_agent_count += 1 + # Settings.json: merge on upgrade (preserve user keys, update hooks), # copy on fresh install if settings_src.is_file(): @@ -266,7 +288,10 @@ def _cmd_install(args: argparse.Namespace) -> None: _copy_file(claude_md_src, claude_md_dst, force=force) action = "Upgraded" if upgrade else "Installed" - print(f"{action}: {agent_count} agents + {ref_count} references to {scope}") + print( + f"{action}: {agent_count} agents + {ref_count} references + " + f"{template_agent_count} agent templates to {scope}" + ) if args.verify: _verify_install(base, agent_target, ref_target, team_ctx) diff --git a/agent_baton/cli/commands/execution/plan_cmd.py b/agent_baton/cli/commands/execution/plan_cmd.py index 1ef5e3da..d0629fe2 100644 --- a/agent_baton/cli/commands/execution/plan_cmd.py +++ b/agent_baton/cli/commands/execution/plan_cmd.py @@ -621,7 +621,14 @@ def handler(args: argparse.Namespace) -> None: print("Planning...", file=sys.stderr) knowledge_registry = KnowledgeRegistry() - knowledge_registry.load_default_paths() + try: + knowledge_registry.load_default_paths(project_root=project_root) + except Exception as exc: + _log.warning( + "Default knowledge registry load failed for `baton plan`; " + "continuing with an empty registry. Cause: %s", + exc, + ) retro_engine = RetrospectiveEngine() bead_store = None @@ -926,7 +933,15 @@ def handler(args: argparse.Namespace) -> None: print("Next: baton execute start") return - if args.json: + if args.explain and args.json: + # Honor both flags: keep the payload machine-parseable and attach + # the explanation instead of silently dropping --json. + payload = plan.to_dict() + payload["explanation"] = planner.explain_plan(plan) + print(json.dumps(payload, indent=2, ensure_ascii=False)) + elif args.explain: + print(planner.explain_plan(plan)) + elif args.json: print(json.dumps(plan.to_dict(), indent=2, ensure_ascii=False)) else: print(plan.to_markdown()) diff --git a/agent_baton/cli/commands/knowledge/doctor_cmd.py b/agent_baton/cli/commands/knowledge/doctor_cmd.py new file mode 100644 index 00000000..b107d14c --- /dev/null +++ b/agent_baton/cli/commands/knowledge/doctor_cmd.py @@ -0,0 +1,663 @@ +"""``baton knowledge`` validation, search, and resolve simulation commands.""" +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +import yaml + +from agent_baton.cli.commands.knowledge import ( + ensure_parent_parser, + register_handler, +) +from agent_baton.core.engine.knowledge_resolver import ( + _DOC_TOKEN_CAP_DEFAULT, + _INLINE_BYTE_THRESHOLD_DEFAULT, + KnowledgeResolver, +) +from agent_baton.core.orchestration.knowledge_registry import KnowledgeRegistry +from agent_baton.core.orchestration.registry import AgentRegistry +from agent_baton.models.knowledge import KnowledgeDocument +from agent_baton.utils.frontmatter import parse_frontmatter + + +_CHARS_PER_TOKEN = 4 + + +@dataclass(frozen=True) +class DoctorIssue: + """Actionable validation issue emitted by ``knowledge doctor``.""" + + severity: str + code: str + message: str + path: str + pack: str + doc: str = "" + + def to_dict(self) -> dict[str, str]: + return { + "severity": self.severity, + "code": self.code, + "message": self.message, + "path": self.path, + "pack": self.pack, + "doc": self.doc, + } + + +def register( + subparsers: argparse._SubParsersAction, # type: ignore[type-arg] +) -> argparse.ArgumentParser: + """Hook doctor/search/resolve into the shared ``baton knowledge`` parser.""" + sub = ensure_parent_parser(subparsers) + + doctor_p = sub.add_parser( + "doctor", + help="Validate knowledge packs and print actionable warnings", + ) + doctor_p.add_argument( + "--knowledge-root", + action="append", + default=None, + help="Knowledge root to validate; repeatable (default: global + project)", + ) + doctor_p.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format: text (default) or json", + ) + doctor_p.add_argument( + "--strict", + action="store_true", + help="Exit non-zero when any warning is found", + ) + doctor_p.add_argument( + "--json", + action="store_const", + const="json", + dest="format", + help="Alias for --format json", + ) + + search_p = sub.add_parser( + "search", + help="Search knowledge metadata with the registry TF-IDF index", + ) + search_p.add_argument("query", nargs="+", help="Search query text") + search_p.add_argument( + "--knowledge-root", + action="append", + default=None, + help="Knowledge root to search; repeatable (default: global + project)", + ) + search_p.add_argument( + "--limit", + type=int, + default=10, + help="Maximum results to return (default: 10)", + ) + search_p.add_argument( + "--format", + choices=("table", "json"), + default="table", + help="Output format: table (default) or json", + ) + search_p.add_argument( + "--json", + action="store_const", + const="json", + dest="format", + help="Alias for --format json", + ) + + resolve_p = sub.add_parser( + "resolve", + help="Simulate knowledge attachments for an agent and task", + ) + resolve_p.add_argument("--agent", required=True, help="Agent name") + resolve_p.add_argument("--task", required=True, help="Task description") + resolve_p.add_argument( + "--knowledge-root", + action="append", + default=None, + help="Knowledge root to load; repeatable (default: global + project)", + ) + resolve_p.add_argument( + "--task-type", + default=None, + help="Optional task type used by resolver keyword extraction", + ) + resolve_p.add_argument( + "--risk", + default="LOW", + help="Risk level passed through to resolver simulation", + ) + resolve_p.add_argument( + "--knowledge-pack", + dest="knowledge_pack", + action="append", + default=[], + help="Explicit pack to include; repeatable", + ) + resolve_p.add_argument( + "--knowledge", + action="append", + default=[], + help="Explicit document path to include; repeatable", + ) + resolve_p.add_argument( + "--format", + choices=("table", "json"), + default="table", + help="Output format: table (default) or json", + ) + resolve_p.add_argument( + "--json", + action="store_const", + const="json", + dest="format", + help="Alias for --format json", + ) + + register_handler("doctor", _run_doctor) + register_handler("search", _run_search) + register_handler("resolve", _run_resolve) + return subparsers.choices["knowledge"] + + +def handler(args: argparse.Namespace) -> None: + """Auto-discovery entry point; delegate to the parent dispatcher.""" + dispatch = getattr(args, "_dispatch", None) + if dispatch is None: + raise SystemExit("baton knowledge: dispatcher missing") + dispatch(args) + + +def _run_doctor(args: argparse.Namespace) -> None: + roots = _knowledge_roots_from_args(args) + explicit = bool(getattr(args, "knowledge_root", None)) + issues, summary = validate_knowledge_roots(roots, require_roots=explicit) + payload = { + "ok": not issues, + "summary": summary, + "issues": [issue.to_dict() for issue in issues], + } + + if getattr(args, "format", "text") == "json": + print(json.dumps(payload, indent=2)) + else: + print(_render_doctor(payload)) + + if getattr(args, "strict", False) and issues: + raise SystemExit(1) + + +def _run_search(args: argparse.Namespace) -> None: + query = " ".join(getattr(args, "query", [])) + limit = max(1, int(getattr(args, "limit", 10) or 10)) + registry = _load_knowledge_registry(getattr(args, "knowledge_root", None)) + results = [ + _search_result_to_dict(registry, doc, score) + for doc, score in registry.search(query, limit=limit) + ] + payload = {"query": query, "results": results} + + if getattr(args, "format", "table") == "json": + print(json.dumps(payload, indent=2)) + else: + print(_render_search(payload)) + + +def _run_resolve(args: argparse.Namespace) -> None: + registry = _load_knowledge_registry(getattr(args, "knowledge_root", None)) + agent_registry = AgentRegistry() + agent_registry.load_default_paths() + resolver = KnowledgeResolver(registry, agent_registry=agent_registry) + attachments = resolver.resolve( + agent_name=args.agent, + task_description=args.task, + task_type=getattr(args, "task_type", None), + risk_level=getattr(args, "risk", "LOW"), + explicit_packs=list(getattr(args, "knowledge_pack", []) or []), + explicit_docs=list(getattr(args, "knowledge", []) or []), + ) + payload = { + "agent": args.agent, + "task": args.task, + "attachments": [attachment.to_dict() for attachment in attachments], + } + + if getattr(args, "format", "table") == "json": + print(json.dumps(payload, indent=2)) + else: + print(_render_resolve(payload)) + + +def validate_knowledge_roots( + roots: Iterable[Path], + *, + require_roots: bool = False, +) -> tuple[list[DoctorIssue], dict[str, int]]: + """Validate all pack directories under *roots*. + + The registry intentionally degrades on bad packs so planning can continue. + Doctor is stricter: it reports the same tolerance points as actionable + edits without changing runtime loading semantics. + + With *require_roots* (explicit ``--knowledge-root`` arguments), a root + that does not exist is itself an issue; default roots are allowed to be + absent. + """ + issues: list[DoctorIssue] = [] + summary = { + "roots": 0, + "packs": 0, + "documents": 0, + "warnings": 0, + } + + if require_roots: + for root in roots: + resolved = root.expanduser() + if not resolved.is_dir(): + issues.append(_issue( + code="missing-root", + path=resolved, + pack="", + message=( + f"Knowledge root '{root}' does not exist or is not a " + "directory. Fix the --knowledge-root argument or " + "create the directory." + ), + )) + + for root in _unique_existing_roots(roots): + summary["roots"] += 1 + for pack_dir in sorted(p for p in root.iterdir() if p.is_dir()): + summary["packs"] += 1 + manifest, manifest_ok = _read_manifest(pack_dir, issues) + default_delivery = str( + manifest.get("default_delivery") or "reference" + ).strip().lower() + + declared_paths = _declared_doc_paths(manifest) + for rel_path in declared_paths: + doc_path = pack_dir / rel_path + if not _declared_doc_exists(pack_dir, rel_path): + issues.append(_issue( + code="missing-declared-file", + path=doc_path, + pack=pack_dir.name, + message=( + f"Pack '{pack_dir.name}' declares missing document " + f"'{rel_path}'. Edit {pack_dir / 'knowledge.yaml'} " + "or create that file." + ), + )) + + if manifest_ok and not str(manifest.get("description") or "").strip(): + issues.append(_issue( + code="empty-pack-description", + path=pack_dir / "knowledge.yaml", + pack=pack_dir.name, + message=( + f"Pack '{pack_dir.name}' has an empty description. " + f"Edit {pack_dir / 'knowledge.yaml'} and add a " + "description for search and resolver matching." + ), + )) + + names: dict[str, list[Path]] = {} + for doc_path in sorted(pack_dir.glob("*.md")): + summary["documents"] += 1 + doc_metadata = _read_doc_metadata( + doc_path, pack_dir.name, issues + ) + if doc_metadata is None: + continue + doc_name, metadata, raw = doc_metadata + names.setdefault(doc_name, []).append(doc_path) + + description = str(metadata.get("description") or "").strip() + if not description: + issues.append(_issue( + code="empty-doc-description", + path=doc_path, + pack=pack_dir.name, + doc=doc_name, + message=( + f"Document '{doc_name}' has an empty description. " + f"Edit {doc_path} frontmatter and add description." + ), + )) + + if _is_large_inline_candidate( + doc_path, raw, default_delivery=default_delivery + ): + issues.append(_issue( + code="large-inline-candidate", + path=doc_path, + pack=pack_dir.name, + doc=doc_name, + message=( + f"Document '{doc_name}' is too large for likely " + f"inline delivery. Edit {doc_path} to shorten it " + f"or edit {pack_dir / 'knowledge.yaml'} and set " + "default_delivery: reference." + ), + )) + + for doc_name, paths in sorted(names.items()): + if len(paths) <= 1: + continue + joined = ", ".join(str(p) for p in paths) + issues.append(_issue( + code="duplicate-doc-name", + path=paths[0], + pack=pack_dir.name, + doc=doc_name, + message=( + f"Document name '{doc_name}' is duplicated in " + f"{joined}. Edit one frontmatter name field so each " + "document name is unique within the pack." + ), + )) + + summary["warnings"] = len(issues) + return issues, summary + + +def _knowledge_roots_from_args(args: argparse.Namespace) -> list[Path]: + raw_roots = getattr(args, "knowledge_root", None) + if raw_roots: + return [Path(root) for root in raw_roots] + return _default_knowledge_roots() + + +def _default_knowledge_roots() -> list[Path]: + return [ + Path.home() / ".claude" / "knowledge", + Path.cwd() / ".claude" / "knowledge", + ] + + +def _unique_existing_roots(roots: Iterable[Path]) -> list[Path]: + seen: set[Path] = set() + existing: list[Path] = [] + for root in roots: + resolved = root.expanduser().resolve() + if resolved in seen or not resolved.is_dir(): + continue + seen.add(resolved) + existing.append(resolved) + return existing + + +def _read_manifest( + pack_dir: Path, + issues: list[DoctorIssue], +) -> tuple[dict[str, Any], bool]: + manifest_path = pack_dir / "knowledge.yaml" + if not manifest_path.is_file(): + issues.append(_issue( + code="missing-manifest", + path=manifest_path, + pack=pack_dir.name, + message=( + f"Pack '{pack_dir.name}' is missing knowledge.yaml. " + f"Edit {manifest_path} and add name, description, tags, " + "target_agents, and default_delivery." + ), + )) + return {}, False + + try: + parsed = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + issues.append(_issue( + code="invalid-manifest", + path=manifest_path, + pack=pack_dir.name, + message=( + f"Pack '{pack_dir.name}' has invalid knowledge.yaml: {exc}. " + f"Edit {manifest_path} and fix the YAML." + ), + )) + return {}, False + + if parsed is None: + return {}, True + + if not isinstance(parsed, dict): + issues.append(_issue( + code="invalid-manifest", + path=manifest_path, + pack=pack_dir.name, + message=( + f"Pack '{pack_dir.name}' knowledge.yaml must be a mapping. " + f"Edit {manifest_path} and use YAML key/value fields." + ), + )) + return {}, False + + return parsed, True + + +def _declared_doc_paths(manifest: dict[str, Any]) -> list[str]: + raw_docs = manifest.get("documents") or manifest.get("docs") or [] + if not isinstance(raw_docs, list): + return [] + + paths: list[str] = [] + for item in raw_docs: + candidate: object + if isinstance(item, str): + candidate = item + elif isinstance(item, dict): + candidate = ( + item.get("path") + or item.get("file") + or item.get("source") + or item.get("name") + ) + else: + candidate = None + if isinstance(candidate, str) and candidate.strip(): + paths.append(candidate.strip()) + return paths + + +def _declared_doc_exists(pack_dir: Path, rel_path: str) -> bool: + doc_path = pack_dir / rel_path + if doc_path.is_file(): + return True + # Only treat the declaration as extensionless when it does not already + # end in .md — Path.suffix would misread dotted stems like "notes.v2". + if rel_path.lower().endswith(".md"): + return False + return doc_path.with_name(doc_path.name + ".md").is_file() + + +def _read_doc_metadata( + doc_path: Path, + pack_name: str, + issues: list[DoctorIssue], +) -> tuple[str, dict[str, Any], str] | None: + try: + raw = doc_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + issues.append(_issue( + code="unreadable-doc", + path=doc_path, + pack=pack_name, + doc=doc_path.stem, + message=( + f"Document '{doc_path.name}' cannot be read: {exc}. " + f"Edit file permissions for {doc_path}." + ), + )) + return None + + metadata, _body = parse_frontmatter(raw) + if not isinstance(metadata, dict): + issues.append(_issue( + code="invalid-doc-frontmatter", + path=doc_path, + pack=pack_name, + doc=doc_path.stem, + message=( + f"Document '{doc_path.stem}' frontmatter must be a mapping. " + f"Edit {doc_path} and use YAML key/value fields." + ), + )) + return None + + if not metadata: + issues.append(_issue( + code="empty-doc-metadata", + path=doc_path, + pack=pack_name, + doc=doc_path.stem, + message=( + f"Document '{doc_path.stem}' has no frontmatter metadata. " + f"Edit {doc_path} and add YAML frontmatter with name, " + "description, tags, grounding, and priority." + ), + )) + + doc_name = str(metadata.get("name") or "").strip() or doc_path.stem + return doc_name, metadata, raw + + +def _is_large_inline_candidate( + doc_path: Path, + raw: str, + *, + default_delivery: str, +) -> bool: + if default_delivery != "inline": + return False + token_estimate = max(1, len(raw) // _CHARS_PER_TOKEN) if raw else 0 + try: + byte_size = doc_path.stat().st_size + except OSError: + byte_size = 0 + return ( + token_estimate > _DOC_TOKEN_CAP_DEFAULT + or byte_size > _INLINE_BYTE_THRESHOLD_DEFAULT + ) + + +def _issue( + *, + code: str, + path: Path, + pack: str, + message: str, + doc: str = "", +) -> DoctorIssue: + return DoctorIssue( + severity="warning", + code=code, + message=message, + path=str(path), + pack=pack, + doc=doc, + ) + + +def _load_knowledge_registry(raw_roots: list[str] | None = None) -> KnowledgeRegistry: + registry = KnowledgeRegistry() + if raw_roots: + for root in raw_roots: + registry.load_directory(Path(root).expanduser(), override=True) + else: + registry.load_default_paths() + return registry + + +def _find_pack_name(registry: KnowledgeRegistry, doc: KnowledgeDocument) -> str: + for pack in registry.all_packs.values(): + for pack_doc in pack.documents: + if pack_doc is doc: + return pack.name + return "" + + +def _search_result_to_dict( + registry: KnowledgeRegistry, + doc: KnowledgeDocument, + score: float, +) -> dict[str, Any]: + return { + "pack": _find_pack_name(registry, doc), + "doc": doc.name, + "score": round(score, 6), + "path": str(doc.source_path) if doc.source_path is not None else "", + "tags": list(doc.tags), + "priority": doc.priority, + "token_estimate": doc.token_estimate, + } + + +def _render_doctor(payload: dict[str, Any]) -> str: + summary = payload["summary"] + lines = [ + "Knowledge doctor", + ( + f"roots={summary['roots']} packs={summary['packs']} " + f"documents={summary['documents']} warnings={summary['warnings']}" + ), + ] + issues = payload["issues"] + if not issues: + lines.append("OK: no knowledge pack warnings found.") + return "\n".join(lines) + for issue in issues: + lines.append( + f"WARNING [{issue['code']}] {issue['message']}" + ) + return "\n".join(lines) + + +def _render_search(payload: dict[str, Any]) -> str: + rows = payload["results"] + if not rows: + return f"No knowledge results for: {payload['query']}" + lines = [ + "| Pack | Document | Score | Priority | Tokens | Path | Tags |", + "|------|----------|-------|----------|--------|------|------|", + ] + for row in rows: + tags = ", ".join(row["tags"]) + lines.append( + f"| {row['pack']} | {row['doc']} | {row['score']:.6f} " + f"| {row['priority']} | {row['token_estimate']} " + f"| {row['path']} | {tags} |" + ) + return "\n".join(lines) + + +def _render_resolve(payload: dict[str, Any]) -> str: + rows = payload["attachments"] + if not rows: + return ( + f"No knowledge attachments resolved for agent '{payload['agent']}'." + ) + lines = [ + "| Source | Pack | Document | Delivery | Retrieval | Tokens | Path |", + "|--------|------|----------|----------|-----------|--------|------|", + ] + for row in rows: + lines.append( + f"| {row['source']} | {row.get('pack_name') or ''} " + f"| {row['document_name']} | {row['delivery']} " + f"| {row['retrieval']} | {row['token_estimate']} " + f"| {row['path']} |" + ) + return "\n".join(lines) diff --git a/agent_baton/cli/commands/quickstart.py b/agent_baton/cli/commands/quickstart.py index 3a2d1c99..8392b311 100644 --- a/agent_baton/cli/commands/quickstart.py +++ b/agent_baton/cli/commands/quickstart.py @@ -214,7 +214,8 @@ def _generate_starter_plan( knowledge_registry = KnowledgeRegistry() try: - knowledge_registry.load_default_paths() + # Load project-scoped packs from the quickstart target repo, not cwd. + knowledge_registry.load_default_paths(project_root=repo_root) except Exception: # KnowledgeRegistry is best-effort -- a fresh project may not have # any documents to load yet. diff --git a/agent_baton/core/engine/executor.py b/agent_baton/core/engine/executor.py index 03e11449..5d320a9e 100644 --- a/agent_baton/core/engine/executor.py +++ b/agent_baton/core/engine/executor.py @@ -2035,7 +2035,9 @@ def start(self, plan: MachinePlan) -> ExecutionAction: state.task_id, exc_info=True, ) - return self._drive_resolver_loop(state) + action = self._drive_resolver_loop(state) + self._save_execution(state) + return action def next_action(self) -> ExecutionAction: """Determine and return the next action based on current state. @@ -2141,6 +2143,9 @@ def next_actions(self) -> list[ExecutionAction]: # Worktree Discipline block and uses relativized paths. wave_isolation = "worktree" if len(dispatchable_steps) >= 2 else "" + team_readiness_before = dict( + state.plan.plan_diagnostics.get("team_readiness", {}) + ) actions: list[ExecutionAction] = [] for step, is_in_flight_team in dispatchable_steps: if step.team: @@ -2160,6 +2165,12 @@ def next_actions(self) -> list[ExecutionAction]: ) ) + if ( + state.plan.plan_diagnostics.get("team_readiness", {}) + != team_readiness_before + ): + self._save_execution(state) + return actions @staticmethod @@ -7803,20 +7814,46 @@ def _team_dispatch_action( leader_member_id=leader.member_id if leader else "", ) + readiness_summary = "" + # A1.a/b: invoke the configured TeamBackend's dispatch hook # exactly once per team step (idempotent: gated on absence of a # parent StepResult). Best-effort — failures never block. if state.get_step_result(step.step_id) is None and self._root is not None: try: - from agent_baton.core.engine.team_backends import select_team_backend + from agent_baton.core.engine.team_backends import ( + UnknownTeamBackendError, + build_team_readiness_diagnostics, + format_team_readiness_summary, + select_team_backend, + write_team_readiness_report, + ) _backend = select_team_backend() + _diagnostics = build_team_readiness_diagnostics( + plan=state.plan, + step=step, + backend_name=_backend.name, + team_context_root=self._root, + ) + _diagnostics = write_team_readiness_report( + diagnostics=_diagnostics, + team_context_root=self._root, + ) + readiness_summary = format_team_readiness_summary(_diagnostics) + _team_diagnostics = dict( + state.plan.plan_diagnostics.get("team_readiness", {}) + ) + _team_diagnostics[step.step_id] = _diagnostics.to_dict() + state.plan.plan_diagnostics["team_readiness"] = _team_diagnostics _backend.on_team_dispatched( plan=state.plan, step=step, team_context_root=self._root, ) + except UnknownTeamBackendError: + raise except Exception as _be_exc: # noqa: BLE001 _log.debug( - "TeamBackend.on_team_dispatched failed (non-fatal): %s", + "TeamBackend readiness/dispatch hook failed (non-fatal): %s", _be_exc, ) @@ -7876,12 +7913,15 @@ def _team_dispatch_action( team_overview=team_overview, prior_beads=_team_beads or None, ) + message = ( + f"Team member '{member.agent_name}' ({member.role}) " + f"for step {step.step_id}." + ) + if readiness_summary: + message = f"{message} {readiness_summary}" member_actions.append(ExecutionAction( action_type=ActionType.DISPATCH, - message=( - f"Team member '{member.agent_name}' ({member.role}) " - f"for step {step.step_id}." - ), + message=message, agent_name=member.agent_name, agent_model=member.model, delegation_prompt=prompt, diff --git a/agent_baton/core/engine/planner.py b/agent_baton/core/engine/planner.py index 91e8030d..d2b334bc 100644 --- a/agent_baton/core/engine/planner.py +++ b/agent_baton/core/engine/planner.py @@ -13,7 +13,11 @@ from __future__ import annotations # --- Public class: pipeline-based planner --- -from agent_baton.core.engine.planning.planner import IntelligentPlanner +from agent_baton.core.engine.planning.planner import ( + IntelligentPlanner, + build_default_knowledge_registry, + ensure_plan_diagnostics, +) # --- GateScope and gate helpers --- from agent_baton.core.engine.planning.utils.gates import ( @@ -44,6 +48,8 @@ __all__ = [ "IntelligentPlanner", + "build_default_knowledge_registry", + "ensure_plan_diagnostics", "GateScope", "_AGENT_ALIASES", "_AGENT_DELIVERABLES", diff --git a/agent_baton/core/engine/planning/planner.py b/agent_baton/core/engine/planning/planner.py index 3dd0beca..612409af 100644 --- a/agent_baton/core/engine/planning/planner.py +++ b/agent_baton/core/engine/planning/planner.py @@ -47,6 +47,9 @@ logger = logging.getLogger(__name__) +_DEFAULT_KNOWLEDGE_REGISTRY = object() + + def _build_default_pipeline() -> Pipeline: """Construct the canonical seven-stage planning pipeline.""" return Pipeline([ @@ -60,6 +63,107 @@ def _build_default_pipeline() -> Pipeline: ]) +def build_default_knowledge_registry(project_root: Path | None = None) -> Any: + """Construct a KnowledgeRegistry and load standard paths. + + If loading fails, return an empty registry and log a clear warning + without surfacing a stack trace to callers. + """ + from agent_baton.core.orchestration.knowledge_registry import KnowledgeRegistry + + registry = KnowledgeRegistry() + try: + registry.load_default_paths(project_root=project_root) + except Exception as exc: + logger.warning( + "Default knowledge registry load failed; continuing with an empty " + "registry. Searched standard knowledge paths only. Cause: %s", + exc, + ) + return registry + + +def build_plan_diagnostics( + plan: "MachinePlan", + *, + knowledge_registry: Any = None, + classification_source: str | None = None, +) -> dict[str, object]: + """Build the public diagnostics payload for an assembled MachinePlan.""" + + def _collect_member_agents(member: object, sink: list[str]) -> None: + agent_name = getattr(member, "agent_name", "") + if agent_name and agent_name != "team": + sink.append(agent_name) + for nested in getattr(member, "sub_team", []) or []: + _collect_member_agents(nested, sink) + + existing = dict(getattr(plan, "plan_diagnostics", {}) or {}) + selected_agent_candidates: list[str] = [ + agent + for agent in list(existing.get("selected_agents", []) or []) + if agent != "team" + ] + for step in plan.all_steps: + if step.agent_name and step.agent_name != "team": + selected_agent_candidates.append(step.agent_name) + for member in getattr(step, "team", []) or []: + _collect_member_agents(member, selected_agent_candidates) + selected_agents = list(dict.fromkeys(selected_agent_candidates)) + knowledge_attachment_count = sum(len(step.knowledge) for step in plan.all_steps) + + if knowledge_registry is not None: + knowledge_packs_loaded = len(knowledge_registry.all_packs) + degraded_packs = sorted(knowledge_registry.degraded_pack_names) + docs_indexed = sum( + len(pack.documents) for pack in knowledge_registry.all_packs.values() + ) + else: + knowledge_packs_loaded = int(existing.get("knowledge_packs_loaded", 0)) + degraded_packs = list(existing.get("degraded_packs", [])) + docs_indexed = int(existing.get("docs_indexed", 0)) + + effective_classification_source = ( + classification_source + or getattr(plan, "classification_source", None) + or existing.get("classification_source") + or "cli-override" + ) + + return { + "task_type": plan.task_type, + "complexity": plan.complexity, + "archetype": plan.archetype, + "risk": plan.risk_level, + "classification_source": effective_classification_source, + "selected_agents": selected_agents, + "phase_count": len(plan.phases), + "gate_count": sum(1 for phase in plan.phases if phase.gate is not None), + "approval_count": sum(1 for phase in plan.phases if phase.approval_required), + "validation_warning_count": int(existing.get("validation_warning_count", 0)), + "knowledge_attachment_count": knowledge_attachment_count, + "knowledge_packs_loaded": knowledge_packs_loaded, + "degraded_packs": degraded_packs, + "docs_indexed": docs_indexed, + "attachments_selected": knowledge_attachment_count, + } + + +def ensure_plan_diagnostics( + plan: "MachinePlan", + *, + knowledge_registry: Any = None, + classification_source: str | None = None, +) -> "MachinePlan": + """Populate ``plan.plan_diagnostics`` with the standard payload.""" + plan.plan_diagnostics = build_plan_diagnostics( + plan, + knowledge_registry=knowledge_registry, + classification_source=classification_source, + ) + return plan + + class IntelligentPlanner: """Pipeline-based planner — standalone implementation. @@ -75,7 +179,7 @@ def __init__( classifier: Any = None, policy_engine: Any = None, retro_engine: Any = None, - knowledge_registry: Any = None, + knowledge_registry: Any = _DEFAULT_KNOWLEDGE_REGISTRY, task_classifier: Any = None, bead_store: Any = None, project_config: Any = None, @@ -84,7 +188,14 @@ def __init__( self._classifier = classifier self._policy_engine = policy_engine self._retro_engine = retro_engine - self.knowledge_registry = knowledge_registry + self._knowledge_registry_is_auto_managed = ( + knowledge_registry is _DEFAULT_KNOWLEDGE_REGISTRY + ) + self.knowledge_registry = ( + build_default_knowledge_registry() + if self._knowledge_registry_is_auto_managed + else knowledge_registry + ) self._bead_store = bead_store # Build collaborators @@ -194,7 +305,9 @@ def create_plan( datetime.now(timezone.utc) if draft.otel_exporter else None ) - services = self._build_services() + services = self._build_services( + knowledge_registry=self._resolve_knowledge_registry(project_root) + ) # Run the pipeline. draft = self._pipeline.run(draft, services) @@ -258,6 +371,14 @@ def explain_plan(self, plan: "MachinePlan") -> str: else: lines.append("No agent health warnings.") + lines.append("") + lines.append("## Plan Diagnostics") + if plan.plan_diagnostics: + for key, value in plan.plan_diagnostics.items(): + lines.append(f"- **{key}**: {value}") + else: + lines.append("No plan diagnostics recorded.") + lines.append("") lines.append("## Routing Notes") if self._last_routing_notes: @@ -598,7 +719,15 @@ def _review_plan_with_llm(self, draft: PlanDraft) -> PlanDraft: return draft - def _build_services(self) -> PlannerServices: + def _resolve_knowledge_registry(self, project_root: Path | None) -> Any: + """Resolve the effective registry for this planning request.""" + if not self._knowledge_registry_is_auto_managed: + return self.knowledge_registry + if project_root is None: + return self.knowledge_registry + return build_default_knowledge_registry(project_root=project_root) + + def _build_services(self, *, knowledge_registry: Any) -> PlannerServices: """Build the services container from this planner's collaborators.""" return PlannerServices( registry=self._registry, @@ -613,7 +742,7 @@ def _build_services(self) -> PlannerServices: data_classifier=self._classifier, policy_engine=self._policy_engine, retro_engine=self._retro_engine, - knowledge_registry=self.knowledge_registry, + knowledge_registry=knowledge_registry, bead_store=self._bead_store, team_context_root=self._team_context_root, ) diff --git a/agent_baton/core/engine/planning/stages/assembly.py b/agent_baton/core/engine/planning/stages/assembly.py index 2e72bab5..7cdfcce8 100644 --- a/agent_baton/core/engine/planning/stages/assembly.py +++ b/agent_baton/core/engine/planning/stages/assembly.py @@ -106,6 +106,7 @@ def _build_shared_context( max_retry_phases=( 3 if draft.planning_archetype == "investigative" else 0 ), + plan_diagnostics=self._build_plan_diagnostics(draft, services), ) # Step 16 — team cost estimation. @@ -134,6 +135,57 @@ def _build_shared_context( tmp_plan.shared_context = shared_context return tmp_plan + def _build_plan_diagnostics( + self, + draft: PlanDraft, + services: PlannerServices, + ) -> dict[str, object]: + """Build a concise, forward-compatible planning diagnostics payload.""" + registry = services.knowledge_registry + knowledge_packs_loaded = len(registry.all_packs) if registry is not None else 0 + degraded_packs = ( + sorted(registry.degraded_pack_names) if registry is not None else [] + ) + docs_indexed = ( + sum(len(pack.documents) for pack in registry.all_packs.values()) + if registry is not None + else 0 + ) + + knowledge_attachment_count = sum( + len(step.knowledge) + for phase in draft.plan_phases + for step in phase.steps + ) + gate_count = sum(1 for phase in draft.plan_phases if phase.gate is not None) + approval_count = sum( + 1 for phase in draft.plan_phases if phase.approval_required + ) + + classification_source = ( + draft.task_classification.source + if draft.task_classification is not None + else "cli-override" + ) + + return { + "task_type": draft.inferred_type, + "complexity": draft.inferred_complexity, + "archetype": draft.planning_archetype, + "risk": draft.risk_level, + "classification_source": classification_source, + "selected_agents": list(draft.resolved_agents), + "phase_count": len(draft.plan_phases), + "gate_count": gate_count, + "approval_count": approval_count, + "validation_warning_count": len(draft.score_warnings), + "knowledge_attachment_count": knowledge_attachment_count, + "knowledge_packs_loaded": knowledge_packs_loaded, + "degraded_packs": degraded_packs, + "docs_indexed": docs_indexed, + "attachments_selected": knowledge_attachment_count, + } + # ------------------------------------------------------------------ # Private: telemetry # ------------------------------------------------------------------ diff --git a/agent_baton/core/engine/planning/stages/decomposition.py b/agent_baton/core/engine/planning/stages/decomposition.py index 557d923b..de73fdb3 100644 --- a/agent_baton/core/engine/planning/stages/decomposition.py +++ b/agent_baton/core/engine/planning/stages/decomposition.py @@ -375,6 +375,7 @@ def _build_investigative_phases( "5. Document: symptoms, timeline, affected paths, reproduction steps\n\n" "Output a structured investigation report with evidence." ), + model="opus", step_type="consulting", )], gate=PlanGate( diff --git a/agent_baton/core/engine/planning/stages/risk.py b/agent_baton/core/engine/planning/stages/risk.py index 08a15409..6da4f025 100644 --- a/agent_baton/core/engine/planning/stages/risk.py +++ b/agent_baton/core/engine/planning/stages/risk.py @@ -25,6 +25,7 @@ from agent_baton.core.engine.planning.services import PlannerServices from agent_baton.core.engine.planning.utils.risk_and_policy import ( assess_risk, + requires_audit_coverage, select_git_strategy, ) from agent_baton.models.enums import RiskLevel @@ -34,12 +35,6 @@ logger = logging.getLogger(__name__) -# Keywords that require the auditor agent regardless of complexity cap. -_AUDIT_KEYWORDS: frozenset[str] = frozenset({ - "compliance", "compliant", "regulated", "regulation", "audit", "auditable", - "gdpr", "hipaa", "sox", "pci", "dss", -}) - class RiskStage: """Stage 3: knowledge setup + risk and sensitivity classification.""" @@ -213,9 +208,12 @@ def _ensure_safety_roster(self, draft: PlanDraft) -> None: if risk is None: return - task_lower = draft.task_summary.lower() needs_reviewer = risk in (RiskLevel.HIGH, RiskLevel.CRITICAL) - needs_auditor = any(kw in task_lower for kw in _AUDIT_KEYWORDS) + needs_auditor = requires_audit_coverage( + draft.task_summary, + draft.classification, + getattr(draft, "policy_violations", None), + ) # Strip stack-flavor suffixes for membership checks (e.g. # "backend-engineer--python" → "backend-engineer"). diff --git a/agent_baton/core/engine/planning/stages/validation.py b/agent_baton/core/engine/planning/stages/validation.py index 867de2f0..78c60245 100644 --- a/agent_baton/core/engine/planning/stages/validation.py +++ b/agent_baton/core/engine/planning/stages/validation.py @@ -6,11 +6,11 @@ **Quality fix #2 — hard gate**: the legacy ``PlanReviewer`` skipped light-complexity plans entirely (``plan_reviewer.py:222``) and treated its findings as advisory. This stage computes a list of *defects* on -top of the reviewer result and exposes them on the draft. Under -``BATON_PLANNER_HARD_GATE`` the stage raises ``PlanQualityError`` -when any defect is critical; without the env var it just records and -warns, preserving legacy behavior so the new gate can bake in -production before flipping the default. +top of the reviewer result and exposes them on the draft. Critical +defects raise ``PlanQualityError`` by default. ``BATON_DEV_MODE=1`` +and ``BATON_PLANNER_WARN_ONLY=1`` make the gate warn-only for local +experimentation; truthy ``BATON_PLANNER_HARD_GATE`` is the legacy +explicit override and blocks even when dev/warn-only mode is set. Defects detected here (independent of what the reviewer surfaces): @@ -21,7 +21,12 @@ 3. **empty_phase** — at least one phase has zero steps. Critical. 4. **agent_phase_mismatch** — a step's agent role is in ``PHASE_BLOCKED_ROLES`` for the phase it landed in. Critical. -5. **reviewer_warning** — the reviewer surfaced any string starting +5. **review_missing** - high-risk or reviewer-routed plans are missing + a Review phase with reviewer coverage. Critical. +6. **audit_missing** - regulated, compliance, policy-auditor, or + auditor-routed plans are missing an Audit phase with auditor coverage. + Critical. +7. **reviewer_warning** — the reviewer surfaced any string starting with ``[critical]``. Critical. Order is preserved: score check + budget tier before consolidation, @@ -29,9 +34,13 @@ """ from __future__ import annotations +import json import logging import os +import re from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace from typing import TYPE_CHECKING from agent_baton.core.engine.planning.draft import PlanDraft @@ -41,13 +50,18 @@ consolidate_team_step, is_team_phase, ) +from agent_baton.core.engine.planning.rules.risk_signals import RISK_ORDINAL from agent_baton.core.engine.planning.utils.risk_and_policy import ( + audit_coverage_requirement, + assess_risk, classify_to_preset_key, select_budget_tier, validate_agents_against_policy, ) from agent_baton.core.engine.planning.utils.roster_logic import check_agent_scores from agent_baton.core.engine.planning.utils.text_parsers import extract_file_paths +from agent_baton.core.orchestration.router import REVIEWER_AGENTS, is_reviewer_agent +from agent_baton.models.enums import RiskLevel if TYPE_CHECKING: from agent_baton.core.govern.classifier import ClassificationResult @@ -57,7 +71,16 @@ class PlanQualityError(RuntimeError): - """Raised by ValidationStage in hard-gate mode when a critical defect is found.""" + """Raised by ValidationStage when the effective quality gate blocks.""" + + def __init__( + self, + message: str, + *, + defects: list["PlanDefect"] | None = None, + ) -> None: + super().__init__(message) + self.defects = list(defects or []) @dataclass @@ -76,15 +99,27 @@ class ValidationStage: """Stage 6: score check, budget tier, plan review with defect detection. Defects are recorded on ``draft.score_warnings`` (any severity) and - on the new ``draft.plan_defects`` attribute (full list). Critical - defects raise ``PlanQualityError`` when ``BATON_PLANNER_HARD_GATE`` - is truthy. + on the new ``draft.plan_defects`` attribute (full list). + Critical defects raise ``PlanQualityError`` by default. Set + ``BATON_DEV_MODE=1`` or ``BATON_PLANNER_WARN_ONLY=1`` to keep local + experimentation warn-only. A truthy ``BATON_PLANNER_HARD_GATE`` remains + a supported legacy explicit opt-in flag and overrides warn-only/dev mode. """ name = "validation" _HARD_GATE_ENV = "BATON_PLANNER_HARD_GATE" + _DEV_MODE_ENV = "BATON_DEV_MODE" + _WARN_ONLY_ENV = "BATON_PLANNER_WARN_ONLY" _TRUTHY = frozenset({"1", "true", "yes", "on"}) - + _IMPLEMENT_PHASE_KEYS = frozenset({ + "implement", + "implementation", + "fix", + "build", + "develop", + "development", + }) + _REVIEWER_BASES = REVIEWER_AGENTS - {"auditor"} def run(self, draft: PlanDraft, services: PlannerServices) -> PlanDraft: # Step 10+11+11b — score check, budget tier, policy validation. # _check_scores writes score_warnings and policy_violations onto draft @@ -103,27 +138,42 @@ def run(self, draft: PlanDraft, services: PlannerServices) -> PlanDraft: # Compute defects from the assembled plan + reviewer result. defects = self._detect_defects(draft) draft.plan_defects = defects # type: ignore[attr-defined] - for d in defects: - if d.severity in ("critical", "warning"): - draft.score_warnings.append(str(d)) + self._apply_quality_gate( + task_id=draft.task_id, + defects=defects, + score_warnings=draft.score_warnings, + ) + return draft + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _apply_quality_gate( + self, + *, + task_id: str, + defects: list[PlanDefect], + score_warnings: list[str] | None = None, + ) -> None: + if score_warnings is not None: + for defect in defects: + if defect.severity in ("critical", "warning"): + score_warnings.append(str(defect)) critical = [d for d in defects if d.severity == "critical"] if critical: logger.warning( "planner.validation: %d critical defect(s) on task %s: %s", - len(critical), draft.task_id, + len(critical), task_id, "; ".join(d.code for d in critical), ) - if self._hard_gate_enabled(): + if self._quality_gate_blocks(): raise PlanQualityError( - f"Plan {draft.task_id} blocked by ValidationStage: " - + "; ".join(str(d) for d in critical[:5]) + f"Plan {task_id} blocked by ValidationStage: " + + "; ".join(str(d) for d in critical[:5]), + defects=critical[:5], ) - return draft - - # ------------------------------------------------------------------ - # Private helpers - # ------------------------------------------------------------------ def _check_scores( self, @@ -233,11 +283,35 @@ def _consolidate_team( # ------------------------------------------------------------------ def _hard_gate_enabled(self) -> bool: - return os.environ.get(self._HARD_GATE_ENV, "").lower() in self._TRUTHY + """Return the legacy explicit hard-gate flag state. + + Plan-quality blocking now defaults on; callers should use + ``_quality_gate_blocks`` for the effective policy. + """ + return self._env_truthy(self._HARD_GATE_ENV) + + def _quality_gate_blocks(self) -> bool: + return self._hard_gate_enabled() or not self._warn_only_enabled() + + def _warn_only_enabled(self) -> bool: + return ( + self._env_truthy(self._DEV_MODE_ENV) + or self._env_truthy(self._WARN_ONLY_ENV) + ) + + def _env_truthy(self, name: str) -> bool: + return os.environ.get(name, "").strip().lower() in self._TRUTHY + + def _with_remediation(self, message: str, remediation: str) -> str: + if re.search(r"\bremediation\s*:", message, flags=re.IGNORECASE): + return message + separator = " " if message.rstrip().endswith((".", "!", "?")) else ". " + return f"{message.rstrip()}{separator}{remediation}" def _detect_defects(self, draft: PlanDraft) -> list[PlanDefect]: """Inspect the assembled draft and return the list of defects.""" defects: list[PlanDefect] = [] + agent_bases = self._agent_bases(draft) # 1. review_skipped review = draft.review_result @@ -248,9 +322,12 @@ def _detect_defects(self, draft: PlanDraft) -> list[PlanDefect]: code="review_skipped", severity="critical", message=( - f"PlanReviewer skipped a {draft.inferred_complexity!r} " - f"plan via the light-complexity early return — " - f"quality gate effectively bypassed." + f"task_id={draft.task_id} source=skipped-light " + f"complexity={draft.inferred_complexity}. " + "PlanReviewer took the light-complexity early return on " + "a non-light plan, bypassing structural review. " + "Remediation: rerun review with the correct complexity " + "or add an explicit Review phase before validation." ), )) # 5. reviewer_warning @@ -259,7 +336,12 @@ def _detect_defects(self, draft: PlanDraft) -> list[PlanDefect]: defects.append(PlanDefect( code="reviewer_warning", severity="critical", - message=w, + message=self._with_remediation( + w, + "Remediation: update the plan to address the " + "reviewer warning, or add explicit Review/Audit " + "coverage that resolves it before validation.", + ), )) # 2. empty_plan @@ -267,23 +349,62 @@ def _detect_defects(self, draft: PlanDraft) -> list[PlanDefect]: defects.append(PlanDefect( code="empty_plan", severity="critical", - message="Plan has zero phases.", + message=( + f"task_id={draft.task_id} phase_count=0. " + "Plan has no executable phases or steps. " + "Remediation: add at least one phase with at least one " + "concrete step before validation." + ), )) return defects + if ( + self._review_required(draft, agent_bases) + and not self._has_review_coverage(draft) + ): + defects.append(PlanDefect( + code="review_missing", + severity="critical", + message=( + f"task_id={draft.task_id} risk={self._risk_value(draft)} " + f"agents={sorted(agent_bases)}. " + "High-risk or reviewer-routed plans require Review coverage. " + "Remediation: add a terminal Review phase with code-reviewer " + "or security-reviewer steps." + ), + )) + + audit_requirement = self._audit_requirement(draft, agent_bases) + if audit_requirement and not self._has_audit_coverage(draft): + defects.append(PlanDefect( + code="audit_missing", + severity="critical", + message=( + f"task_id={draft.task_id} agents={sorted(agent_bases)} " + f"{audit_requirement}. " + "Compliance or auditor-routed plans require Audit coverage. " + "Remediation: add a terminal Audit phase with an auditor step." + ), + )) + for phase in draft.plan_phases: # 3. empty_phase if not phase.steps: defects.append(PlanDefect( code="empty_phase", severity="critical", - message=f"Phase {phase.name!r} has zero steps.", + message=( + f"task_id={draft.task_id} phase_id={phase.phase_id} " + f"phase={phase.name!r} step_count=0. " + "Phase has no executable steps. " + "Remediation: add at least one step to this phase or " + "remove the empty phase." + ), )) continue # 4. agent_phase_mismatch - phase_key = (phase.name or "").lower().split(":")[0].strip() - phase_key = phase_key.split()[-1] if phase_key else "" + phase_key = self._phase_key(phase.name) blocked = PHASE_BLOCKED_ROLES.get(phase_key, set()) if blocked: for step in phase.steps: @@ -293,10 +414,262 @@ def _detect_defects(self, draft: PlanDraft) -> list[PlanDefect]: code="agent_phase_mismatch", severity="critical", message=( - f"Step {step.step_id} routes " - f"{base!r} into the blocked-list phase " - f"{phase.name!r}." + f"task_id={draft.task_id} phase_id={phase.phase_id} " + f"phase={phase.name!r} step_id={step.step_id} " + f"agent={base!r}. Agent is blocked from this " + "phase type. Remediation: route the step to an " + "allowed implementer for this phase or move the " + "agent to a dedicated Review/Audit phase." ), )) + if phase_key in self._IMPLEMENT_PHASE_KEYS: + for step in phase.steps: + for base in self._step_agent_bases(step): + if is_reviewer_agent(base): + target_phase = ( + "Audit" if base == "auditor" else "Review" + ) + defects.append(PlanDefect( + code="agent_phase_mismatch", + severity="critical", + message=( + f"task_id={draft.task_id} phase_id={phase.phase_id} " + f"phase={phase.name!r} step_id={step.step_id} " + f"agent={base!r}. Reviewer-class agents are " + "blocked from implementation phases. " + f"Remediation: move {base!r} to a dedicated " + f"{target_phase} phase and assign implementation " + "work to an engineering agent." + ), + )) return defects + + def _review_required(self, draft: PlanDraft, agent_bases: set[str]) -> bool: + return ( + self._risk_value(draft) in {"HIGH", "CRITICAL"} + or bool(agent_bases & self._REVIEWER_BASES) + ) + + def _audit_required(self, draft: PlanDraft, agent_bases: set[str]) -> bool: + return self._audit_requirement(draft, agent_bases) is not None + + def _audit_requirement(self, draft: PlanDraft, agent_bases: set[str]) -> str | None: + if "auditor" in agent_bases: + return "requirement=auditor_routed" + return audit_coverage_requirement( + draft.task_summary, + getattr(draft, "classification", None), + getattr(draft, "policy_violations", None), + ) + + def _has_review_coverage(self, draft: PlanDraft) -> bool: + for phase in draft.plan_phases: + if self._phase_key(phase.name) != "review": + continue + for step in phase.steps: + if self._step_agent_bases(step) & self._REVIEWER_BASES: + return True + return False + + def _has_audit_coverage(self, draft: PlanDraft) -> bool: + for phase in draft.plan_phases: + if self._phase_key(phase.name) != "audit": + continue + for step in phase.steps: + if "auditor" in self._step_agent_bases(step): + return True + return False + + def _risk_value(self, draft: PlanDraft) -> str: + risk = getattr(draft, "risk_level_enum", None) + if isinstance(risk, RiskLevel): + return risk.value + if risk: + return str(risk).upper() + return str(getattr(draft, "risk_level", "") or "").upper() + + def _phase_key(self, name: str) -> str: + raw = (name or "").lower().split(":")[0].strip() + key = raw.split()[-1] if raw else "" + if key == "implementation": + return "implement" + if key == "development": + return "develop" + return key + + def _agent_bases(self, draft: PlanDraft) -> set[str]: + bases = { + (agent or "").split("--")[0] + for agent in getattr(draft, "resolved_agents", []) or [] + if agent + } + for phase in draft.plan_phases: + for step in phase.steps: + bases.update(self._step_agent_bases(step)) + bases.discard("") + return bases + + def _step_agent_bases(self, step: object) -> set[str]: + bases: set[str] = set() + agent_name = getattr(step, "agent_name", "") + if agent_name: + bases.add(agent_name.split("--")[0]) + for member in getattr(step, "team", []) or []: + member_name = getattr(member, "agent_name", "") + if member_name: + bases.add(member_name.split("--")[0]) + for nested in getattr(member, "sub_team", []) or []: + nested_name = getattr(nested, "agent_name", "") + if nested_name: + bases.add(nested_name.split("--")[0]) + return bases + + +def _resolved_agents_from_plan(plan: "MachinePlan") -> list[str]: + agents: list[str] = [] + + def _append(agent_name: str) -> None: + if agent_name and agent_name != "team": + agents.append(agent_name) + + def _collect_member_agents(member: object) -> None: + _append(str(getattr(member, "agent_name", "") or "")) + for nested in getattr(member, "sub_team", []) or []: + _collect_member_agents(nested) + + for step in plan.all_steps: + _append(step.agent_name) + for member in getattr(step, "team", []) or []: + _collect_member_agents(member) + + return list(dict.fromkeys(agents)) + + +def _classification_from_plan(plan: "MachinePlan") -> object | None: + if not plan.classification_signals: + return None + try: + payload = json.loads(plan.classification_signals) + except (TypeError, ValueError): + return None + if not isinstance(payload, dict): + return None + + risk_level = str(payload.get("risk_level") or plan.risk_level or "LOW").upper() + try: + risk_enum = RiskLevel(risk_level) + except ValueError: + risk_enum = RiskLevel.LOW + + confidence = "high" if (plan.classification_confidence or 0.0) >= 1.0 else "medium" + return SimpleNamespace( + signals_found=list(payload.get("signals") or []), + risk_level=risk_enum, + guardrail_preset=str(payload.get("guardrail_preset") or "Standard Development"), + explanation=str(payload.get("explanation") or ""), + confidence=confidence, + ) + + +def _classification_payload(classification: object) -> dict[str, object]: + risk = getattr(classification, "risk_level", RiskLevel.LOW) + if isinstance(risk, RiskLevel): + risk_value = risk.value + else: + risk_value = str(risk or RiskLevel.LOW.value).upper() + return { + "signals": list(getattr(classification, "signals_found", []) or []), + "risk_level": risk_value, + "guardrail_preset": str( + getattr(classification, "guardrail_preset", "Standard Development") + or "Standard Development" + ), + "explanation": str(getattr(classification, "explanation", "") or ""), + } + + +def _classification_confidence_value(classification: object) -> float: + return 1.0 if getattr(classification, "confidence", "") == "high" else 0.5 + + +def _coerce_risk_level(value: object, default: RiskLevel = RiskLevel.LOW) -> RiskLevel: + if isinstance(value, RiskLevel): + return value + try: + return RiskLevel(str(value or "").upper()) + except ValueError: + return default + + +def _sync_plan_validation(plan: "MachinePlan", draft: PlanDraft) -> None: + plan.risk_level = draft.risk_level + plan.phases = list(draft.plan_phases) + plan.budget_tier = draft.budget_tier + existing = dict(getattr(plan, "plan_diagnostics", {}) or {}) + existing["validation_warning_count"] = max( + int(existing.get("validation_warning_count", 0)), + len(draft.score_warnings), + ) + plan.plan_diagnostics = existing + + +def validate_assembled_plan( + plan: "MachinePlan", + *, + services: PlannerServices, + project_root: Path | None = None, +) -> "MachinePlan": + """Apply ValidationStage semantics to an already-assembled MachinePlan.""" + draft = PlanDraft.from_inputs( + plan.task_summary, + task_type=plan.task_type, + complexity=plan.complexity, + project_root=project_root, + explicit_knowledge_packs=list(plan.explicit_knowledge_packs or []), + explicit_knowledge_docs=list(plan.explicit_knowledge_docs or []), + intervention_level=plan.intervention_level, + ) + draft.task_id = plan.task_id + draft.plan_phases = list(plan.phases) + draft.resolved_agents = _resolved_agents_from_plan(plan) + draft.inferred_type = plan.task_type or "" + draft.inferred_complexity = plan.complexity or "medium" + draft.risk_level = str(plan.risk_level or "") + draft.git_strategy = plan.git_strategy + draft.classification = _classification_from_plan(plan) + if draft.classification is None and services.data_classifier is not None: + try: + draft.classification = services.data_classifier.classify(plan.task_summary) + except Exception: + draft.classification = None + else: + plan.classification_signals = json.dumps( + _classification_payload(draft.classification) + ) + plan.classification_confidence = _classification_confidence_value( + draft.classification + ) + + risk_candidates = [ + _coerce_risk_level(plan.risk_level), + _coerce_risk_level(assess_risk(plan.task_summary, draft.resolved_agents)), + ] + if draft.classification is not None: + risk_candidates.append( + _coerce_risk_level(getattr(draft.classification, "risk_level", None)) + ) + draft.risk_level_enum = max( + risk_candidates, + key=lambda risk: RISK_ORDINAL[risk], + ) + draft.risk_level = draft.risk_level_enum.value + + stage = ValidationStage() + try: + stage.run(draft, services) + except PlanQualityError: + _sync_plan_validation(plan, draft) + raise + _sync_plan_validation(plan, draft) + return plan diff --git a/agent_baton/core/engine/planning/utils/risk_and_policy.py b/agent_baton/core/engine/planning/utils/risk_and_policy.py index 772d87da..c2cf4350 100644 --- a/agent_baton/core/engine/planning/utils/risk_and_policy.py +++ b/agent_baton/core/engine/planning/utils/risk_and_policy.py @@ -26,6 +26,23 @@ logger = logging.getLogger(__name__) +# Keywords that imply audit coverage even when no classifier is available. +AUDIT_COVERAGE_TERMS: frozenset[str] = frozenset({ + "audit", + "auditable", + "compliant", + "compliance", + "dss", + "gdpr", + "hipaa", + "pci", + "regulated", + "regulation", + "regulatory", + "sox", +}) + + # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- @@ -109,6 +126,50 @@ def classify_to_preset_key(classification: "ClassificationResult | None") -> str return mapping.get(name, "standard_dev") +def audit_coverage_requirement( + task_summary: str, + classification: "ClassificationResult | None", + policy_violations: object | None = None, +) -> str | None: + """Return the reason a task requires audit coverage, if any.""" + preset = str(getattr(classification, "guardrail_preset", "") or "").strip() + if preset.lower() == "regulated data": + return f"guardrail_preset={preset}" + + for violation in policy_violations or []: + rule = getattr(violation, "rule", None) + if ( + str(getattr(rule, "rule_type", "")).lower() == "require_agent" + and str(getattr(rule, "pattern", "")).split("--")[0] == "auditor" + and str(getattr(rule, "severity", "")).lower() == "block" + ): + rule_name = str( + getattr(rule, "name", "require_agent") or "require_agent" + ) + return f"policy_violation={rule_name}" + + summary = (task_summary or "").lower() + if any( + re.search(rf"\b{re.escape(term)}\b", summary) + for term in AUDIT_COVERAGE_TERMS + ): + return "requirement=compliance_signal" + return None + + +def requires_audit_coverage( + task_summary: str, + classification: "ClassificationResult | None", + policy_violations: object | None = None, +) -> bool: + """Return whether the task requires a dedicated auditor/Audit phase.""" + return audit_coverage_requirement( + task_summary, + classification, + policy_violations, + ) is not None + + def validate_agents_against_policy( agents: list[str], policy_set: "PolicySet", diff --git a/agent_baton/core/engine/team_backends.py b/agent_baton/core/engine/team_backends.py index 82f20c03..8b8582fb 100644 --- a/agent_baton/core/engine/team_backends.py +++ b/agent_baton/core/engine/team_backends.py @@ -41,12 +41,15 @@ """ from __future__ import annotations +from dataclasses import dataclass, replace +import json import logging import os from pathlib import Path from typing import Protocol, runtime_checkable from agent_baton.models.execution import MachinePlan, PlanStep +from agent_baton.utils.frontmatter import parse_frontmatter _log = logging.getLogger(__name__) @@ -142,6 +145,68 @@ def hook_record_command( # can coordinate effectively. Larger teams should be split. _MAX_TEAM_MEMBERS = 5 +CLAUDE_TEAMS_CAVEATS: tuple[str, ...] = ( + "no resume: baton execute resume cannot revive in-flight Claude Teams teammates", + "no nesting: nested teams are flattened for Claude Teams dispatch", + "one team at a time: a lead session can coordinate only one Agent Team at a time", + "fixed permissions: teammate permissions are fixed at spawn time", + "missing skills/MCP frontmatter: subagent skills and mcpServers frontmatter are not honored for teammates", +) + + +class UnknownTeamBackendError(ValueError): + """Raised when strict backend selection rejects an unknown backend.""" + + +@dataclass(frozen=True) +class TeamReadinessDiagnostics: + """Structured readiness summary emitted before a team step is dispatched.""" + + backend: str + step_id: str + member_count: int + top_level_member_count: int + nested_team_count: int + shared_files: list[str] + shared_contracts: list[dict[str, object]] + synthesis_strategy: str + conflict_strategy: str + warnings: list[str] + report_path: str = "" + + @property + def warning_count(self) -> int: + return len(self.warnings) + + def to_dict(self) -> dict[str, object]: + return { + "backend": self.backend, + "step_id": self.step_id, + "member_count": self.member_count, + "top_level_member_count": self.top_level_member_count, + "nested_team_count": self.nested_team_count, + "shared_files": list(self.shared_files), + "shared_contracts": [dict(c) for c in self.shared_contracts], + "synthesis_strategy": self.synthesis_strategy, + "conflict_strategy": self.conflict_strategy, + "warning_count": self.warning_count, + "warnings": list(self.warnings), + "report_path": self.report_path, + } + + def with_report_path(self, report_path: str) -> "TeamReadinessDiagnostics": + return replace(self, report_path=report_path) + + +def _dedupe(values: list[str] | tuple[str, ...]) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for value in values: + if value and value not in seen: + seen.add(value) + out.append(value) + return out + def _flatten_members(team: list["TeamMember"]) -> list["TeamMember"]: """Depth-first flatten of a team roster, descending into ``sub_team``. @@ -418,11 +483,129 @@ def _render_spawn_prompt( "(`baton execute resume` cannot revive an in-flight team).\n" "- One team at a time; clean up before another team step.\n" "- No nested teams (sub-teams are flattened above).\n" + "- Fixed permissions at spawn time; adjust permissions before launch.\n" "- Subagent `skills` / `mcpServers` frontmatter is NOT honored on " "teammates.\n" ) +# --------------------------------------------------------------------------- +# Readiness diagnostics + report artifact +# --------------------------------------------------------------------------- + +def build_team_readiness_diagnostics( + *, + plan: MachinePlan, + step: PlanStep, + backend_name: str, + team_context_root: Path | None = None, +) -> TeamReadinessDiagnostics: + """Build the runtime readiness payload for a team step. + + The payload is deliberately small and serializable so it can be written to + ``team-report.json``, copied into ``MachinePlan.plan_diagnostics``, and + summarized in dispatch messages without changing plan schema. + """ + flat_members = _flatten_members(step.team) + nested_team_count = sum(1 for member in flat_members if member.sub_team) + synthesis = step.synthesis + warnings: list[str] = [] + + if backend_name == ClaudeTeamsBackend.name: + # Step-specific warnings come first: the dispatch summary caps + # warning_notes, and the static caveats would otherwise consume it. + if nested_team_count: + warnings.append( + f"team has {nested_team_count} nested team(s); claude-teams " + "will flatten the structure" + ) + if len(step.team) > _MAX_TEAM_MEMBERS: + warnings.append( + f"team has {len(step.team)} top-level members; recommended " + f"maximum is {_MAX_TEAM_MEMBERS}" + ) + if team_context_root is not None: + safety_flags = ClaudeTeamsBackend._audit_step_agents( + step, team_context_root, + ) + for agent_name, fields in sorted(safety_flags.items()): + warnings.append( + f"agent {agent_name} declares {'/'.join(fields)} " + "frontmatter; claude-teams teammates will miss it" + ) + warnings.extend(CLAUDE_TEAMS_CAVEATS) + + shared_contracts = [ + { + "member_id": member.member_id, + "agent_name": member.agent_name, + "role": member.role, + "task_description": member.task_description, + "deliverables": list(member.deliverables), + } + for member in flat_members + ] + + return TeamReadinessDiagnostics( + backend=backend_name, + step_id=step.step_id, + member_count=len(flat_members), + top_level_member_count=len(step.team), + nested_team_count=nested_team_count, + shared_files=_dedupe(list(step.context_files or [])), + shared_contracts=shared_contracts, + synthesis_strategy=( + synthesis.strategy if synthesis is not None else "concatenate" + ), + conflict_strategy=( + synthesis.conflict_handling if synthesis is not None else "auto_merge" + ), + warnings=warnings, + ) + + +def write_team_readiness_report( + *, + diagnostics: TeamReadinessDiagnostics, + team_context_root: Path, +) -> TeamReadinessDiagnostics: + """Write ``team-report.json`` under the existing team artifact directory.""" + team_dir = team_context_root / "teams" / f"team-{diagnostics.step_id}" + team_dir.mkdir(parents=True, exist_ok=True) + report = team_dir / "team-report.json" + try: + report_path = report.relative_to(team_context_root).as_posix() + except ValueError: + report_path = report.as_posix() + diagnostics = diagnostics.with_report_path(report_path) + report.write_text( + json.dumps(diagnostics.to_dict(), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return diagnostics + + +def format_team_readiness_summary( + diagnostics: TeamReadinessDiagnostics, +) -> str: + """Return a one-line summary safe for existing CLI/API message fields.""" + parts = [ + f"Team readiness: backend={diagnostics.backend}", + f"members={diagnostics.member_count}", + f"nested={diagnostics.nested_team_count}", + f"shared_files={len(diagnostics.shared_files)}", + f"contracts={len(diagnostics.shared_contracts)}", + f"synthesis={diagnostics.synthesis_strategy}", + f"conflict={diagnostics.conflict_strategy}", + f"warnings={diagnostics.warning_count}", + ] + if diagnostics.report_path: + parts.append(f"report={diagnostics.report_path}") + if diagnostics.warnings: + parts.append("warning_notes=[" + "; ".join(diagnostics.warnings[:5]) + "]") + return "; ".join(parts) + "." + + # --------------------------------------------------------------------------- # Selector # --------------------------------------------------------------------------- @@ -433,6 +616,15 @@ def _render_spawn_prompt( } +def _strict_backend_selection_enabled() -> bool: + return os.environ.get("BATON_TEAMS_BACKEND_STRICT", "0").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + + def select_team_backend(name: str | None = None) -> TeamBackend: """Return the configured TeamBackend instance. @@ -441,16 +633,21 @@ def select_team_backend(name: str | None = None) -> TeamBackend: 2. ``BATON_TEAMS_BACKEND`` env var. 3. Default: ``"worktree"``. - Unknown values log a warning and fall back to ``"worktree"`` so a - typo in settings never breaks execution. + Unknown values log a warning and fall back to ``"worktree"`` by default + so a typo in settings never breaks execution. Set + ``BATON_TEAMS_BACKEND_STRICT=1`` to fail instead. """ chosen = (name or os.environ.get("BATON_TEAMS_BACKEND") or "worktree").strip().lower() cls = _BACKENDS.get(chosen) if cls is None: + msg = ( + f"Unknown BATON_TEAMS_BACKEND={chosen!r}. " + f"Valid choices: {sorted(_BACKENDS)}" + ) + if _strict_backend_selection_enabled(): + raise UnknownTeamBackendError(msg) _log.warning( - "Unknown BATON_TEAMS_BACKEND=%r; falling back to 'worktree'. " - "Valid choices: %s", - chosen, sorted(_BACKENDS), + "%s; falling back to 'worktree'.", msg, ) cls = WorktreeTeamBackend return cls() @@ -467,10 +664,9 @@ def audit_agents_for_teammate_safety( YAML frontmatter declares load-bearing fields that Claude Code Agent Teams does NOT honor when the agent is used as a teammate. - Read the .md files in *agents_dir*, parse the YAML header (a thin - inline parser — no PyYAML dep so the helper stays cheap to import), - and flag any agent with non-empty ``skills:`` or ``mcpServers:`` - fields. + Read the .md files in *agents_dir*, parse the YAML header through the + shared frontmatter utility, and flag any agent with non-empty + ``skills:`` or ``mcpServers:`` values. """ flagged: dict[str, list[str]] = {} if not agents_dir.exists(): @@ -480,30 +676,30 @@ def audit_agents_for_teammate_safety( text = md.read_text(encoding="utf-8") except OSError: continue - # Frontmatter starts and ends with --- on its own line. - if not text.startswith("---"): - continue - try: - _, fm, _ = text.split("---", 2) - except ValueError: + metadata, _body = parse_frontmatter(text) + if not isinstance(metadata, dict): continue agent_name = md.stem problems: list[str] = [] for field in ("skills", "mcpServers"): - # Look for "field:" at start of a line, followed by a - # non-empty value or a yaml list. - for line in fm.splitlines(): - stripped = line.strip() - if stripped.startswith(f"{field}:"): - rhs = stripped.split(":", 1)[1].strip() - if rhs and rhs not in ("[]", "{}", "null", "~"): - problems.append(field) - break + if field in metadata and _frontmatter_value_is_non_empty(metadata[field]): + problems.append(field) if problems: flagged[agent_name] = problems return flagged +def _frontmatter_value_is_non_empty(value: object) -> bool: + """Return True when a parsed frontmatter value declares real content.""" + if value is None: + return False + if isinstance(value, str): + return bool(value.strip()) + if isinstance(value, (list, tuple, set, dict)): + return bool(value) + return bool(value) + + def check_resumability_constraints(plan: MachinePlan) -> list[str]: """Return planner-time warnings for the claude-teams backend (A1.d). @@ -557,6 +753,11 @@ def check_resumability_constraints(plan: MachinePlan) -> list[str]: "TeamBackend", "WorktreeTeamBackend", "ClaudeTeamsBackend", + "TeamReadinessDiagnostics", + "UnknownTeamBackendError", + "build_team_readiness_diagnostics", + "format_team_readiness_summary", + "write_team_readiness_report", "select_team_backend", "audit_agents_for_teammate_safety", "check_resumability_constraints", diff --git a/agent_baton/core/orchestration/knowledge_registry.py b/agent_baton/core/orchestration/knowledge_registry.py index c3f60983..c542be4a 100644 --- a/agent_baton/core/orchestration/knowledge_registry.py +++ b/agent_baton/core/orchestration/knowledge_registry.py @@ -367,39 +367,54 @@ def load_directory(self, directory: Path, *, override: bool = False) -> int: return 0 count = 0 - for pack_dir in sorted(directory.iterdir()): - if not pack_dir.is_dir(): - continue - loaded = self._load_pack(pack_dir) - if loaded is None: - continue - pack, degraded = loaded - if override or pack.name not in self._packs: - # Remove stale TF-IDF entries for the overridden pack, if any. - # Simplest approach: rebuild index entries after each override. - self._packs[pack.name] = pack - if degraded: - self._degraded_pack_names.add(pack.name) - else: - self._degraded_pack_names.discard(pack.name) - count += 1 - - # Rebuild TF-IDF index from scratch whenever new packs are added. - self._rebuild_tfidf() + try: + for pack_dir in sorted(directory.iterdir()): + if not pack_dir.is_dir(): + continue + try: + loaded = self._load_pack(pack_dir) + except Exception as exc: + logger.warning( + "Skipping knowledge pack %s after load failure: %s", + pack_dir, + exc, + ) + continue + if loaded is None: + continue + pack, degraded = loaded + if override or pack.name not in self._packs: + # Remove stale TF-IDF entries for the overridden pack, if any. + # Simplest approach: rebuild index entries after each override. + self._packs[pack.name] = pack + if degraded: + self._degraded_pack_names.add(pack.name) + else: + self._degraded_pack_names.discard(pack.name) + count += 1 + finally: + # Rebuild TF-IDF from whatever loaded before any pack-level failure. + self._rebuild_tfidf() return count - def load_default_paths(self) -> int: + def load_default_paths(self, project_root: Path | None = None) -> int: """Load packs from standard locations (global then project override). Mirrors AgentRegistry.load_default_paths(): - Global: ``~/.claude/knowledge/`` - - Project: ``.claude/knowledge/`` (relative to cwd, resolved) + - Project: ``/.claude/knowledge/`` when *project_root* + is provided, otherwise ``.claude/knowledge/`` relative to cwd Returns: Total number of packs loaded. """ global_dir = Path.home() / ".claude" / "knowledge" - project_dir = (Path(".claude") / "knowledge").resolve() + if project_root is None: + project_dir = (Path(".claude") / "knowledge").resolve() + else: + project_dir = ( + Path(project_root).expanduser().resolve() / ".claude" / "knowledge" + ) count = self.load_directory(global_dir) count += self.load_directory(project_dir, override=True) @@ -513,8 +528,19 @@ def _load_pack(self, pack_dir: Path) -> tuple[KnowledgePack, bool] | None: if manifest_path.is_file(): try: raw = manifest_path.read_text(encoding="utf-8") - manifest = yaml.safe_load(raw) or {} - except (OSError, yaml.YAMLError) as exc: + parsed = yaml.safe_load(raw) + if parsed is None: + manifest = {} + elif isinstance(parsed, dict): + manifest = parsed + else: + logger.warning( + "Failed to parse %s: manifest must be a mapping", + manifest_path, + ) + manifest = {} + degraded = True + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: logger.warning("Failed to parse %s: %s", manifest_path, exc) manifest = {} degraded = True diff --git a/agent_baton/core/pmo/forge.py b/agent_baton/core/pmo/forge.py index c7789c34..08984fc2 100644 --- a/agent_baton/core/pmo/forge.py +++ b/agent_baton/core/pmo/forge.py @@ -26,6 +26,11 @@ from agent_baton.core.pmo.store import PmoStore from agent_baton.core.runtime.headless import HeadlessClaude +from agent_baton.core.engine.planner import ( + build_default_knowledge_registry, + ensure_plan_diagnostics, +) +from agent_baton.core.engine.planning.stages.validation import validate_assembled_plan from agent_baton.models.execution import MachinePlan from agent_baton.models.pmo import InterviewQuestion, InterviewAnswer, PmoProject @@ -70,6 +75,30 @@ def __init__( self._session_started: str | None = None self._plans_created: int = 0 + def _finalize_headless_plan( + self, + plan: MachinePlan, + *, + project_root: Path | None, + ) -> MachinePlan: + resolver = getattr(self._planner, "_resolve_knowledge_registry", None) + if callable(resolver): + knowledge_registry = resolver(project_root) + else: + knowledge_registry = build_default_knowledge_registry(project_root=project_root) + services = self._planner._build_services(knowledge_registry=knowledge_registry) + validate_assembled_plan( + plan, + services=services, + project_root=project_root, + ) + ensure_plan_diagnostics( + plan, + knowledge_registry=knowledge_registry, + classification_source=plan.classification_source, + ) + return plan + def create_plan( self, description: str, @@ -130,6 +159,7 @@ def create_plan( ) if plan is not None: plan.classification_source = "headless-claude" + self._finalize_headless_plan(plan, project_root=project_root) self._plans_created += 1 logger.info("Forge: plan generated via headless Claude") return plan @@ -336,6 +366,7 @@ def regenerate_plan( ) if plan is not None: plan.classification_source = "headless-claude" + self._finalize_headless_plan(plan, project_root=project_root) logger.info("Forge: regenerated plan via headless Claude") return plan logger.warning("Forge: headless regen failed, falling back to IntelligentPlanner") diff --git a/agent_baton/core/runtime/headless.py b/agent_baton/core/runtime/headless.py index d35b6bd0..c56f4c65 100644 --- a/agent_baton/core/runtime/headless.py +++ b/agent_baton/core/runtime/headless.py @@ -592,6 +592,9 @@ def _build_plan_prompt( "- Every code-producing phase MUST have a test gate (pytest).", "- Research/investigate/review phases do NOT need gates.", "- Use 'opus' model only for complex architectural decisions. Default to 'sonnet'.", + "- HIGH or CRITICAL risk plans MUST include a Review phase staffed by code-reviewer or security-reviewer.", + "- Regulated-data, compliance, or audit-sensitive plans MUST include an Audit phase whose final word is 'Audit' and whose step is staffed by the agent literally named auditor.", + "- Reviewer-class agents (code-reviewer, security-reviewer, auditor) are banned from implementation/fix/build/develop phases.", "- task_id should be a URL-safe slug: lowercase, hyphens, max 60 chars.", "- step_id format: '.' (e.g. '1.1', '2.1', '2.2').", ]) diff --git a/agent_baton/core/storage/active_task.py b/agent_baton/core/storage/active_task.py new file mode 100644 index 00000000..b0685385 --- /dev/null +++ b/agent_baton/core/storage/active_task.py @@ -0,0 +1,90 @@ +"""Read-only active-task probes for project ``baton.db`` files.""" +from __future__ import annotations + +import shutil +import sqlite3 +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class ActiveTaskProbe: + """Result from a read-only active-task lookup.""" + + task_id: str | None + db_path: Path + degraded: bool = False + error: str | None = None + error_type: str | None = None + + def degradation_details(self) -> dict[str, Any]: + return { + "status": "degraded", + "db_path": str(self.db_path), + "error": self.error or "", + "error_type": self.error_type or "", + } + + +def read_active_task_id_from_db_copy(db_path: Path) -> ActiveTaskProbe: + """Read the active task ID from a temp copy of ``baton.db``. + + The source database may be in WAL mode. Opening it directly, even with + ``mode=ro``, can create ``-wal`` and ``-shm`` sidecars next to the project + database. Copying the main database into a temp directory confines any + SQLite side effects to that temp directory. + """ + if not db_path.is_file(): + return ActiveTaskProbe(task_id=None, db_path=db_path) + + try: + with tempfile.TemporaryDirectory(prefix="baton-db-read-") as temp_dir: + db_copy = Path(temp_dir) / db_path.name + shutil.copy2(db_path, db_copy) + for suffix in ("-wal", "-shm"): + sidecar = db_path.with_name(f"{db_path.name}{suffix}") + if sidecar.is_file(): + shutil.copy2(sidecar, db_copy.with_name(f"{db_copy.name}{suffix}")) + conn = sqlite3.connect( + f"{db_copy.resolve().as_uri()}?mode=ro", + uri=True, + ) + try: + has_active_task = conn.execute( + ( + "SELECT 1 FROM sqlite_master " + "WHERE type = 'table' AND name = 'active_task' LIMIT 1" + ) + ).fetchone() + if not has_active_task: + return ActiveTaskProbe(task_id=None, db_path=db_path) + row = conn.execute( + ( + "SELECT task_id FROM active_task " + "WHERE id = 1 LIMIT 1" + ) + ).fetchone() + finally: + conn.close() + except Exception as exc: + # Deliberate exception to the "never swallow OperationalError" rule: + # this read-only diagnostic probe surfaces failures as a degraded + # ActiveTaskProbe in doctor check details instead of raising. + return ActiveTaskProbe( + task_id=None, + db_path=db_path, + degraded=True, + error=str(exc), + error_type=type(exc).__name__, + ) + + if not row: + return ActiveTaskProbe(task_id=None, db_path=db_path) + active_task_id = row[0] + if isinstance(active_task_id, str): + active_task_id = active_task_id.strip() + if active_task_id: + return ActiveTaskProbe(task_id=active_task_id, db_path=db_path) + return ActiveTaskProbe(task_id=None, db_path=db_path) diff --git a/agent_baton/core/storage/schema.py b/agent_baton/core/storage/schema.py index abe8be77..866e9a08 100644 --- a/agent_baton/core/storage/schema.py +++ b/agent_baton/core/storage/schema.py @@ -40,7 +40,7 @@ current ``SCHEMA_VERSION``. """ -SCHEMA_VERSION = 46 +SCHEMA_VERSION = 47 # Sequential migration scripts: {version: DDL_string} MIGRATIONS: dict[int, str] = { @@ -1399,6 +1399,18 @@ -- Applied to BOTH project and central databases via -- ConnectionManager._run_migrations() (see v45's note above). ALTER TABLE plans ADD COLUMN manager_mode INTEGER NOT NULL DEFAULT 0; +""", + 47: """ +-- v47: persist plan diagnostics as a JSON blob on plans. +-- +-- Applied to BOTH project and central databases. Project DBs have a single +-- task_id primary key; central DBs have (project_id, task_id). The additive +-- column shape works for both and keeps MachinePlan.to_dict()/from_dict() +-- diagnostics round-trips intact after SQLite normalization. +-- +-- Renumbered from 46 on merge: master's v46 (manager_mode) was already +-- published, so databases migrated by master must still receive this column. +ALTER TABLE plans ADD COLUMN plan_diagnostics TEXT NOT NULL DEFAULT '{}'; """, } @@ -1554,6 +1566,7 @@ task_type TEXT, classification_signals TEXT, classification_confidence REAL, + plan_diagnostics TEXT NOT NULL DEFAULT '{}', -- release_id: soft FK to releases.release_id (R3.1). Not declared as a -- hard REFERENCES because the v16 migration (ALTER TABLE ADD COLUMN) -- cannot add an FK in SQLite, and migrated vs. fresh DBs must behave @@ -2669,6 +2682,7 @@ task_type TEXT, classification_signals TEXT, classification_confidence REAL, + plan_diagnostics TEXT NOT NULL DEFAULT '{}', release_id TEXT, -- v46 (M9): manager-mode PMO layer flag -- see MIGRATIONS[46] above. manager_mode INTEGER NOT NULL DEFAULT 0, diff --git a/agent_baton/core/storage/sqlite_backend.py b/agent_baton/core/storage/sqlite_backend.py index 046c460a..4e20e0d5 100644 --- a/agent_baton/core/storage/sqlite_backend.py +++ b/agent_baton/core/storage/sqlite_backend.py @@ -2218,8 +2218,8 @@ def _upsert_plan(conn: sqlite3.Connection, plan: "MachinePlan") -> None: # noqa explicit_knowledge_packs, explicit_knowledge_docs, intervention_level, task_type, classification_signals, classification_confidence, - manager_mode) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + manager_mode, plan_diagnostics) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( plan.task_id, @@ -2239,6 +2239,7 @@ def _upsert_plan(conn: sqlite3.Connection, plan: "MachinePlan") -> None: # noqa plan.classification_signals, plan.classification_confidence, int(plan.manager_mode), + json.dumps(plan.plan_diagnostics), ), ) @@ -2483,6 +2484,12 @@ def _load_plan_struct( # pre-v46 databases (defaults to False, matching MachinePlan's own # field default). mm = plan_row["manager_mode"] if "manager_mode" in plan_keys else 0 + # v47: plan diagnostics blob, graceful fallback for pre-v47 databases. + pd = ( + plan_row["plan_diagnostics"] + if "plan_diagnostics" in plan_keys + else "{}" + ) return MachinePlan( task_id=plan_row["task_id"], @@ -2502,4 +2509,5 @@ def _load_plan_struct( classification_signals=cs, classification_confidence=float(cc) if cc is not None else None, manager_mode=bool(mm), + plan_diagnostics=json.loads(pd or "{}"), ) diff --git a/agent_baton/models/execution.py b/agent_baton/models/execution.py index f32421e4..baacc6c6 100644 --- a/agent_baton/models/execution.py +++ b/agent_baton/models/execution.py @@ -616,6 +616,7 @@ class MachinePlan(PlanModel): archetype: str = "phased" max_retry_phases: int = 0 compliance_fail_closed: bool | None = None + plan_diagnostics: dict[str, Any] = Field(default_factory=dict) # Goal-driven execution (G1, see docs/internal/agent-teams-and-goal-design.md). # Set when the plan is created via `baton goal ""`. The engine # invokes a GoalEvaluator at phase boundaries and uses amend_plan() to @@ -759,6 +760,8 @@ def to_dict(self) -> dict: "max_amend_cycles": self.max_amend_cycles, "manager_mode": self.manager_mode, } + if self.plan_diagnostics: + d["plan_diagnostics"] = dict(self.plan_diagnostics) if self.resource_limits is not None: d["resource_limits"] = self.resource_limits.to_dict() return d diff --git a/agents/talent-builder.md b/agents/talent-builder.md index 33d1a69a..762703cd 100644 --- a/agents/talent-builder.md +++ b/agents/talent-builder.md @@ -150,6 +150,49 @@ knowledge/[domain]/ **File:** `.claude/agents/[name].md` or `~/.claude/agents/[name].md` +**Generated-Agent Contract:** + +Every generated agent must follow this contract. Use +`references/agent-authoring.md` as the durable reference and +`.claude/templates/agents/*.md` as the starter source. + +Starter template files: +- `.claude/templates/agents/base-agent.md` +- `.claude/templates/agents/flavored-agent.md` +- `.claude/templates/agents/reviewer-agent.md` + +Required frontmatter fields: +- `name` +- `description` +- `model` +- `permissionMode` +- `tools` + +Recommended frontmatter fields: +- `owner` +- `status` +- `version` +- `created_by` +- `last_reviewed` +- `knowledge_packs` + +Required body sections: +- Mission +- Before Starting +- Knowledge References +- Principles +- Anti-Patterns +- Output Format + +Reference and tool rules: +- Avoid broad tools unless the agent mission requires them. Start read-only + (`Read`, `Glob`, `Grep`) for reviewers and researchers; add `Edit`, `Write`, + or `Bash` only when the agent must mutate files or run commands. +- Read back every generated file before reporting it as complete. +- Validate references exist before saving the agent. Every knowledge pack, + reference doc, skill, or template path named in the agent must resolve, or + the agent must explicitly state why it is optional. + **Template:** ```markdown @@ -163,14 +206,31 @@ permissionMode: [auto-edit for implementers, default for reviewers] color: [unused color] tools: [minimum needed — Read, Glob, Grep for read-only; add Write, Edit, Bash for implementers] +owner: [team or person responsible for maintenance] +status: draft +version: 0.1.0 +created_by: talent-builder +last_reviewed: [YYYY-MM-DD] +knowledge_packs: + - [knowledge/domain/overview.md or remove if none] --- # [Role Title] +## Mission + You are a [seniority + role]. [One-sentence mission.] ## Before Starting +1. Read this entire agent definition. +2. Read back every file listed under "Knowledge References"; do not rely on + stale memory. +3. Validate references exist and are relevant before using them. If a + reference is missing, report the gap. + +## Knowledge References + Read these knowledge packs before doing any work: - [path to knowledge pack files relevant to this agent] @@ -208,9 +268,19 @@ Return: **Agent quality checklist:** - [ ] Description is specific enough to trigger correctly -- [ ] Knowledge pack paths are referenced in "Before Starting" +- [ ] Required frontmatter fields exist: `name`, `description`, `model`, + `permissionMode`, `tools` +- [ ] Recommended frontmatter fields are filled when ownership is known: + `owner`, `status`, `version`, `created_by`, `last_reviewed`, + `knowledge_packs` +- [ ] Required body sections exist: Mission, Before Starting, Knowledge + References, Principles, Anti-Patterns, Output Format +- [ ] Knowledge pack paths are referenced in "Knowledge References" - [ ] Baked-in knowledge is concise (< 100 lines of domain content) -- [ ] Tools are minimum needed (principle of least privilege) +- [ ] Avoid broad tools; tools are minimum needed (principle of least + privilege) +- [ ] Read back the final agent file and validate references before reporting + completion - [ ] Output format matches the orchestrator's expectations - [ ] For flavored variants: references base role, same output format diff --git a/docs/agent-roster.md b/docs/agent-roster.md index 5cd29b4b..d1bae43e 100644 --- a/docs/agent-roster.md +++ b/docs/agent-roster.md @@ -4,11 +4,15 @@ audience: agents, maintainers see-also: - [orchestrator-usage.md](orchestrator-usage.md) - [../references/agent-routing.md](../references/agent-routing.md) + - [../references/agent-authoring.md](../references/agent-authoring.md) - [cli-reference.md](cli-reference.md#baton-agents) --- # Agent roster +!!! abstract "Pillar context" + This page details **Pillar 2 — Compose the right team**. For the high-level map of all four pillars, see [The Four Pillars](pillars.md). + This page mirrors `agents/*.md` — the distributable agent definitions installed by `scripts/install.sh`. There are **30** agents. The orchestrator picks among them based on task domain, risk tier, and budget. To dispatch one directly inside Claude Code, name it in the `Agent` tool with `subagent_type`. To inspect runtime registration: `baton agents`. @@ -22,7 +26,11 @@ Agents do not run on their own. They run when: 1. The **orchestrator agent** dispatches them as part of a baton-driven plan, OR 2. A user (or another agent) names them via Claude Code's `Agent` tool with a `subagent_type` parameter. -Each agent file in `agents/` contains a YAML frontmatter block (`name`, `description`, `model`, `tools`) and a body prompt. The frontmatter is what the runtime registers; the body is what the agent reads when dispatched. +Agent files in `agents/` use a YAML frontmatter block and a body prompt. The +frontmatter is what the runtime registers; the body is what the agent reads +when dispatched. New or updated/generated agents should include `name`, +`description`, `model`, `permissionMode`, and `tools`, and should follow the +generated-agent contract in [`references/agent-authoring.md`](../references/agent-authoring.md). ## Orchestration & routing @@ -96,4 +104,4 @@ Each agent file in `agents/` contains a YAML frontmatter block (`name`, `descrip --- -For the routing logic that picks among these agents, see [`references/agent-routing.md`](../references/agent-routing.md). For risk-tier guardrails, see [`references/guardrail-presets.md`](../references/guardrail-presets.md). For each agent's full prompt, read the matching file in `agents/.md`. +For the routing logic that picks among these agents, see [`references/agent-routing.md`](../references/agent-routing.md). For the generated-agent authoring contract, see [`references/agent-authoring.md`](../references/agent-authoring.md). For risk-tier guardrails, see [`references/guardrail-presets.md`](../references/guardrail-presets.md). For each agent's full prompt, read the matching file in `agents/.md`. diff --git a/docs/architecture.md b/docs/architecture.md index 7e6ffb78..b9430b96 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,6 +11,25 @@ see-also: This page explains *why* Agent Baton is built the way it is. For component-level structure see [`architecture/high-level-design.md`](architecture/high-level-design.md). For internals (state machine, planner, executor, dispatcher, gates, persistence) see [`architecture/technical-design.md`](architecture/technical-design.md). For the package map with `path:line` cites see [`architecture/package-layout.md`](architecture/package-layout.md). For the action enum and transitions see [`architecture/state-machine.md`](architecture/state-machine.md). +!!! abstract "Pillar context" + This overview covers the engine behind all four pillars. For the high-level map of what Baton is *for*, see [The Four Pillars](pillars.md). + +## How this maps to the four pillars + +Baton is a project manager for Claude Code. Every architectural decision below serves one of four pillars: + +- **Pillar 1 — Plan with foresight.** The planner, risk classifier, and spec federation pipeline turn a natural-language task into a structured, risk-tiered, budget-aware plan before a single agent runs. +- **Pillar 2 — Compose the right team.** The agent registry, talent-builder, and dispatcher match each phase to the specialist best suited for it, keeping context scoped and fresh. +- **Pillar 3 — Right agent, right problem, right time.** The execution engine, PMO UI, and orchestration loop sequence agents, manage state, and recover from crashes without losing progress. +- **Pillar 4 — Checks & balances.** Gates, assurance packs, the auditor agent, and compliance audit trails enforce correctness and auditability at every phase boundary. + +```mermaid +flowchart LR + T(["Task"]) --> P1["1 · Plan with foresight"] --> P2["2 · Compose the right team"] --> P3["3 · Right agent, right time"] --> P4["4 · Checks & balances"] --> D(["Tested, auditable code"]) +``` + +This overview is the engine story. For the what-and-why of each pillar, see [The Four Pillars](pillars.md). + ## What problem Baton solves Long Claude Code sessions on cross-cutting tasks tend to: lose context between subtasks, miss test coverage because gating depends on the operator remembering, leave no audit trail, and have no way to recover when the session crashes. Baton adds a project management layer that breaks work into phases, scopes each phase to one specialist agent, enforces automated gates, and persists state so a crashed session resumes cleanly. diff --git a/docs/architecture/high-level-design.md b/docs/architecture/high-level-design.md index 2e7cacde..be867b9d 100644 --- a/docs/architecture/high-level-design.md +++ b/docs/architecture/high-level-design.md @@ -9,10 +9,13 @@ --- +!!! abstract "Pillar context" + The components described here implement the four pillars. For the high-level map, see [The Four Pillars](../pillars.md). + ## 1. System overview -Agent Baton is a Python orchestration engine that drives Claude Code -subagents through structured execution plans. A single `agent_baton/` +Agent Baton is a project manager for Claude Code that drives subagents +through structured execution plans. A single `agent_baton/` package provides three coequal interfaces — CLI, HTTP API, and a React PMO frontend — over a shared engine and storage layer. @@ -64,7 +67,7 @@ distributable assets: | Artifact | Location | Purpose | |----------|----------|---------| | `agent_baton/` Python package | `pyproject.toml` editable install | The engine, CLI, API, and bundled agents | -| `agents/` markdown | Installed to `~/.claude/agents/` by `scripts/install.sh` | 30 agent definitions | +| `agents/` markdown | Installed to `~/.claude/agents/` by `scripts/install.sh` | 30 agent definitions (Pillar 2) | | `references/` markdown | Installed to `~/.claude/references/` | 19 procedure references | | `templates/` | Installed to project's `.claude/` | `CLAUDE.md` + `settings.json` + skills | | `pmo-ui/dist/` | Built and served at `/pmo/` by FastAPI | React PMO frontend | @@ -77,8 +80,8 @@ The `baton` console-script is registered by `pyproject.toml`. |-----------|------|-------| | `baton` CLI | Console script | [`agent_baton/cli/main.py`](../../agent_baton/cli/main.py) | | HTTP API | FastAPI app | [`agent_baton/api/server.py`](../../agent_baton/api/server.py) | -| PMO frontend | React/Vite SPA | [`pmo-ui/`](../../pmo-ui/) | -| Execution engine | State machine | [`agent_baton/core/engine/executor.py`](../../agent_baton/core/engine/executor.py) | +| PMO frontend | React/Vite SPA | [`pmo-ui/`](../../pmo-ui/) (Pillar 3) | +| Execution engine | State machine | [`agent_baton/core/engine/executor.py`](../../agent_baton/core/engine/executor.py) (Pillar 3) | | Async runtime | `asyncio` driver | [`agent_baton/core/runtime/worker.py`](../../agent_baton/core/runtime/worker.py) | | Per-project store | SQLite + JSON | `.claude/team-context/baton.db` (+ legacy JSON) | | Federated store | SQLite (read replica) | `~/.baton/central.db` | diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 8c2cbd42..f9e0fe44 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -19,8 +19,9 @@ Commands are organized into functional groups: | **Execution** | Plan, execute, and manage orchestrated tasks | `plan`, `execute`, `status`, `daemon`, `async`, `decide` | | **Manager Mode** | PMO planning overlay -- charter, team blueprint, scope contracts, knowledge-pack lifecycle | `plan --manager-mode`, `config`, `report`, `team`, `knowledge list/scan/show/audit/propose` | | **Observe** | Traces, usage, dashboards, telemetry | `dashboard`, `trace`, `usage`, `telemetry`, `context-profile`, `retro`, `context`, `cleanup` | -| **Govern** | Risk, policy, compliance, validation | `classify`, `compliance`, `policy`, `escalations`, `validate`, `spec-check`, `detect` | +| **Guardrails** | Risk, policy, compliance, validation | `classify`, `compliance`, `policy`, `escalations`, `validate`, `spec-check`, `detect` | | **Improve** | Scoring, learning, patterns, budgets | `scores`, `learn`, `patterns`, `budget`, `changelog`, `anomalies` | +| **Knowledge** | Knowledge pack validation, search, briefing, lifecycle, effectiveness | `knowledge doctor`, `knowledge search`, `knowledge resolve`, `knowledge brief`, `knowledge harvest`, `knowledge stale`, `knowledge deprecate`, `knowledge retire`, `knowledge sweep`, `knowledge usage`, `knowledge effectiveness`, `knowledge ranking`, `knowledge ab` | | **Distribute** | Packaging, publishing, installation | `package`, `publish`, `pull`, `install`, `transfer` | | **Agents** | Agent discovery, routing, events | `agents`, `route`, `events`, `incident` | | **PMO** | Portfolio management overlay | `pmo serve`, `pmo status`, `pmo add`, `pmo health` | @@ -1061,7 +1062,7 @@ baton cleanup --retention-days 60 --- -## Govern Commands +## Guardrails Commands ### `baton classify` @@ -1842,6 +1843,271 @@ baton learn run-cycle --run --- +## Knowledge Commands + +### `baton knowledge` + +Knowledge utilities: validate packs, search metadata, simulate resolver +attachments, generate codebase briefings, harvest knowledge from existing +artefacts, manage item lifecycle, and report effectiveness. This is a +command group with subcommands. + +``` +baton knowledge SUBCOMMAND [options] +``` + +| Subcommand | Description | +|------------|-------------| +| `doctor` | Validate knowledge packs and print actionable warnings | +| `search` | Search knowledge metadata with the registry TF-IDF index | +| `resolve` | Simulate knowledge attachments for an agent and task | +| `brief` | Generate a concise codebase briefing for new agents | +| `harvest` | Harvest knowledge entries from existing artefacts (ADRs, PR reviews) | +| `stale` | List active knowledge items that look stale | +| `deprecate` | Flag a knowledge item as deprecated | +| `retire` | Retire a knowledge item immediately (skips grace window) | +| `sweep` | Auto-retire deprecated items whose grace period has elapsed | +| `usage` | Show usage count and last-used time for a single item | +| `effectiveness` | Show per-doc effectiveness + ROI scores | +| `ranking` | Show all known docs ranked by historical effectiveness | +| `ab` | Manage knowledge A/B experiments | + +#### `baton knowledge doctor` + +Validate knowledge packs and print actionable warnings (missing or +invalid `knowledge.yaml`, declared documents that don't exist, empty +descriptions, duplicate document names, documents too large for inline +delivery). Runtime loading tolerates these issues; doctor reports them +as actionable edits without changing loading semantics. + +``` +baton knowledge doctor [--knowledge-root DIR ...] [--format text|json] [--json] [--strict] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--knowledge-root DIR` | global + project | Knowledge root to validate; repeatable (default: `~/.claude/knowledge` and `./.claude/knowledge`) | +| `--format FORMAT` | `text` | Output format: `text` or `json` | +| `--json` | -- | Alias for `--format json` | +| `--strict` | false | Exit non-zero when any warning is found | + +#### `baton knowledge search` + +Search knowledge metadata with the registry TF-IDF index. + +``` +baton knowledge search QUERY... [options] +``` + +| Argument | Required | Default | Description | +|----------|----------|---------|-------------| +| `QUERY` | Yes | -- | Search query text (one or more words) | +| `--knowledge-root DIR` | No | global + project | Knowledge root to search; repeatable | +| `--limit N` | No | `10` | Maximum results to return | +| `--format FORMAT` | No | `table` | Output format: `table` or `json` | +| `--json` | No | -- | Alias for `--format json` | + +#### `baton knowledge resolve` + +Simulate the knowledge attachments the resolver would produce for an +agent and task, without dispatching anything. + +``` +baton knowledge resolve --agent NAME --task TEXT [options] +``` + +| Flag | Required | Default | Description | +|------|----------|---------|-------------| +| `--agent NAME` | Yes | -- | Agent name | +| `--task TEXT` | Yes | -- | Task description | +| `--knowledge-root DIR` | No | global + project | Knowledge root to load; repeatable | +| `--task-type TYPE` | No | -- | Optional task type used by resolver keyword extraction | +| `--risk LEVEL` | No | `LOW` | Risk level passed through to resolver simulation | +| `--knowledge-pack PACK` | No | -- | Explicit pack to include; repeatable | +| `--knowledge PATH` | No | -- | Explicit document path to include; repeatable | +| `--format FORMAT` | No | `table` | Output format: `table` or `json` | +| `--json` | No | -- | Alias for `--format json` | + +#### `baton knowledge brief` + +Generate a concise codebase briefing (stack, layout, entry points, +conventions, tests, health snapshot) that dispatched agents can read +instead of re-discovering the basics every run. + +``` +baton knowledge brief [--project DIR] [--save] [--format markdown|json] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--project DIR` | cwd | Project directory to brief | +| `--save` | false | Write to `.claude/team-context/codebase-brief.md` (or `codebase-brief.json` with `--format json`) instead of stdout | +| `--format FORMAT` | `markdown` | Output format: `markdown` or `json` | + +#### `baton knowledge harvest` + +Convert existing artefacts into knowledge entries. Both harvesters are +idempotent: re-running on unchanged input is a no-op. + +``` +baton knowledge harvest adrs [--source-dir DIR] [--target-pack PACK] [--knowledge-root DIR] +baton knowledge harvest reviews --pr N [--repo OWNER/NAME] [--knowledge-root DIR] +``` + +`adrs` walks a docs tree for Architecture Decision Records and converts +each into a knowledge document: + +| Flag | Default | Description | +|------|---------|-------------| +| `--source-dir DIR` | `docs/` | Root to walk for ADR markdown files | +| `--target-pack PACK` | `decisions` | Knowledge pack name to write into | +| `--knowledge-root DIR` | `.claude/knowledge` | Override the knowledge root | + +`reviews` pulls PR review comments via the `gh` CLI and distils salient +ones into a lessons document: + +| Flag | Required | Default | Description | +|------|----------|---------|-------------| +| `--pr N` | Yes | -- | Pull-request number to harvest | +| `--repo OWNER/NAME` | No | auto-detect via `gh` | Repository slug | +| `--knowledge-root DIR` | No | `.claude/knowledge` | Override the knowledge root | + +#### `baton knowledge stale` + +List active knowledge items that look stale. Lifecycle subcommands +(`stale`, `deprecate`, `retire`, `sweep`, `usage`) operate on the +project database at `.claude/team-context/baton.db`. + +``` +baton knowledge stale [--days N] [--max-usage N] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--days N` | `90` | Days since last use threshold | +| `--max-usage N` | `5` | Items with a usage count below this are eligible | + +#### `baton knowledge deprecate` + +Flag a knowledge item as deprecated; schedules retirement after the +grace period elapses. + +``` +baton knowledge deprecate KNOWLEDGE_ID [--grace N] [--reason TEXT] +``` + +| Argument | Required | Default | Description | +|----------|----------|---------|-------------| +| `KNOWLEDGE_ID` | Yes | -- | Item ID in the form `/` | +| `--grace N` | No | `30` | Grace period in days before auto-retirement | +| `--reason TEXT` | No | -- | Optional human-readable reason recorded with the deprecation | + +#### `baton knowledge retire` + +Retire a knowledge item immediately, skipping the grace window. + +``` +baton knowledge retire KNOWLEDGE_ID +``` + +#### `baton knowledge sweep` + +Auto-retire deprecated items whose grace period has elapsed. Safe to +schedule (e.g. in a daily cron) — it only retires items the operator +has already deprecated. + +``` +baton knowledge sweep +``` + +#### `baton knowledge usage` + +Show lifecycle state, usage count, last-used time, and staleness for a +single item. + +``` +baton knowledge usage KNOWLEDGE_ID +``` + +#### `baton knowledge effectiveness` + +Show per-doc effectiveness + ROI scores as a sorted Markdown table or +JSON document. + +``` +baton knowledge effectiveness [--pack PACK] [--since-days N] [--format markdown|json] [--stale] [--threshold-days N] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--pack PACK` | all packs | Restrict the report to a single knowledge pack | +| `--since-days N` | `30` | Rolling window in days for the effectiveness rollup (`0` = all time) | +| `--format FORMAT` | `markdown` | Output format: `markdown` or `json` | +| `--stale` | false | Show only stale candidates | +| `--threshold-days N` | `90` | Stale-by-age threshold in days (used with `--stale`) | + +#### `baton knowledge ranking` + +Show all known docs ranked by historical effectiveness. Reads +`v_knowledge_effectiveness` from the central database. + +``` +baton knowledge ranking [--output table|json] [--db PATH] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `--output FORMAT` | `table` | Output format: `table` or `json` | +| `--db PATH` | `~/.baton/central.db` | Path to the SQLite database | + +#### `baton knowledge ab` + +Manage knowledge A/B experiments. + +``` +baton knowledge ab list +baton knowledge ab create --kid KNOWLEDGE_ID --a PATH_A --b PATH_B [--ratio RATIO] +baton knowledge ab results EXPERIMENT_ID +baton knowledge ab stop EXPERIMENT_ID +``` + +`create` flags: + +| Flag | Required | Default | Description | +|------|----------|---------|-------------| +| `--kid KNOWLEDGE_ID` | Yes | -- | Canonical pack/doc id (e.g. `security/owasp.md`) | +| `--a PATH_A` | Yes | -- | Relative path to the variant A document | +| `--b PATH_B` | Yes | -- | Relative path to the variant B document | +| `--ratio RATIO` | No | `0.5` | Fraction routed to variant A (between 0.0 and 1.0) | + +**Examples:** + +```bash +# Validate knowledge packs, failing CI on warnings +baton knowledge doctor --strict + +# Search the knowledge index +baton knowledge search payment idempotency --limit 5 + +# Preview what the resolver would attach for a dispatch +baton knowledge resolve --agent backend-engineer--python \ + --task "Add JWT authentication middleware" --risk MEDIUM + +# Generate and save a codebase brief +baton knowledge brief --save + +# Harvest ADRs into the decisions pack +baton knowledge harvest adrs --source-dir docs + +# Lifecycle: deprecate with a 14-day grace period, then sweep later +baton knowledge deprecate security/owasp --grace 14 --reason "superseded" +baton knowledge sweep +``` + +**Related:** `baton plan --knowledge / --knowledge-pack`, `baton learn` + +--- + ## Distribute Commands ### `baton package` diff --git a/docs/engine-and-runtime.md b/docs/engine-and-runtime.md index 87875ffc..6ca1441a 100644 --- a/docs/engine-and-runtime.md +++ b/docs/engine-and-runtime.md @@ -1,5 +1,8 @@ # Execution Engine and Runtime +!!! abstract "Pillar context" + This page details **Pillar 1 — Plan with foresight** (and **Pillar 3 — Right agent, right problem, right time**). For the high-level map of all four pillars, see [The Four Pillars](pillars.md). + This document is the authoritative reference for Agent Baton's execution engine (`agent_baton/core/engine/`) and runtime system (`agent_baton/core/runtime/`). It covers the complete lifecycle of an diff --git a/docs/governance-knowledge-and-events.md b/docs/governance-knowledge-and-events.md index 765df3a1..0b09e11a 100644 --- a/docs/governance-knowledge-and-events.md +++ b/docs/governance-knowledge-and-events.md @@ -1,7 +1,10 @@ # Knowledge, Events, and Governance How knowledge delivery, event-driven traceability, and risk guardrails -support reliable multi-agent orchestration. +keep every agent action informed, traceable, and trustworthy. + +!!! abstract "Pillar context" + This page is primarily **Pillar 4 — Checks & balances** (Governance is the trust mechanism that enforces risk policy and produces audit artifacts); Knowledge serves Pillars 1–2 by informing planning and team assembly, and Events support Pillar 3 by powering orchestration traceability and crash recovery. For the high-level map of all four pillars, see [The Four Pillars](pillars.md). --- diff --git a/docs/index.md b/docs/index.md index 4e16e011..12db84b0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ **Turn one prompt into a coordinated team of AI specialists.** -Agent Baton is a multi-agent orchestration system for [Claude Code](https://docs.anthropic.com/en/docs/claude-code). Describe a complex task in plain language — Baton plans it, routes it to the right specialist agents, enforces QA gates between phases, and delivers tested, reviewed code. No external services. No API keys beyond Claude. Everything runs locally. +Agent Baton is a project manager for [Claude Code](https://docs.anthropic.com/en/docs/claude-code). Describe a complex effort in plain language — Baton plans it with foresight, composes the right specialist agents, dispatches each to the right problem at the right time, and keeps the work on track with checks and balances. It delivers tested, reviewed code. No external services. No API keys beyond Claude. Everything runs locally. ``` You: "Use the orchestrator to add input validation to the API @@ -15,22 +15,35 @@ Baton: Plans 3 phases (implement, test, review) Writes trace, usage log, and retrospective ``` +## What Baton is for + +Four high-level goals, in priority order: + +1. **Plan with foresight** — break the effort down, classify risk, and forecast cost *before* execution, so you see where it will break up front. +2. **Compose the right team** — create bespoke, narrowly-scoped specialists (via `talent-builder`) so no generalist drowns in whole-codebase context. +3. **Right agent, right problem, right time** — a deterministic engine dispatches each phase to the specialist that fits it, with QA gates between. +4. **Checks & balances** — an independent `auditor` (and a `subject-matter-expert` for regulated work) verifies the result is *functionally* right, not just that it lints. + +→ See **[The Four Pillars](pillars.md)** for the full picture and a visual of how an effort flows through them. + ## Why Agent Baton? -Claude Code is powerful, but complex tasks — the ones that touch multiple files, need testing, and require different expertise — benefit from structure. Without it, you get context bloat, missed test coverage, and no audit trail. +Claude Code is powerful, but a complex effort — one that spans many files, needs different kinds of expertise, and has to actually be *correct* — is hard to run as one long conversation. You get context rot, missed coverage, no foresight into what's coming, and no record of what happened. -Agent Baton gives Claude Code a project management layer. It breaks work into phases, assigns each phase to a specialist agent, runs automated QA gates between them, and tracks everything. You stay in control while the agents do the heavy lifting. +Agent Baton gives Claude Code a project-management layer. You stay in control while the agents do the heavy lifting. | Without Baton | With Baton | |---------------|------------| -| One long conversation doing everything | Phases with specialist agents | -| Manual "did you run the tests?" | Automated pytest/lint gates between phases | +| One long conversation doing everything | A planned effort, phased and sequenced | +| One generalist drowning in whole-codebase context | Bespoke specialists tuned to each problem | +| No idea what's coming or what might break | Up-front plan, risk classification, cost forecast | +| Manual "did you run the tests?" | Automated gates + domain-expert checks between phases | +| Hope the AI got it *right* | Independent auditor / SME verification on risky work | | No record of what happened | Full traces, usage logs, retrospectives | -| Hope the AI remembers context | Scoped delegation prompts per agent | -| Single point of failure | Crash recovery via `baton execute resume` | ## Where to go next +- **[The Four Pillars](pillars.md)** — what Baton is for, with a visual of the workflow - **[Orchestrator Usage](orchestrator-usage.md)** — how to drive a task end-to-end through the engine - **[Agent Roster](agent-roster.md)** — the 30 specialist agents Baton can dispatch - **[Architecture Overview](architecture.md)** — the orchestration engine, storage, and supporting subsystems diff --git a/docs/observe-learn-and-improve.md b/docs/observe-learn-and-improve.md index df793fc3..02c76cc0 100644 --- a/docs/observe-learn-and-improve.md +++ b/docs/observe-learn-and-improve.md @@ -3,8 +3,11 @@ Agent Baton includes a closed-loop learning pipeline that automatically collects execution data, discovers patterns in that data, and proposes (or auto-applies) improvements to future plans. The three subsystems -- -Observe, Learn, and Improve -- form a feedback cycle that makes the -orchestration system better with every task it runs. +Observe, Learn, and Improve -- form a feedback cycle that supports better +execution across every task. + +!!! abstract "Pillar context" + This page covers the observability and learning toolkit that supports all four pillars: traces and retrospectives feed Pillar 1 (plan with foresight), agent scorecards and roster recommendations inform Pillar 2 (compose the right team), event-driven dashboards support Pillar 3 (right agent, right problem, right time), and compliance artifacts underpin Pillar 4 (checks & balances). For the high-level map of all four pillars, see [The Four Pillars](pillars.md). ``` THE LEARNING LOOP diff --git a/docs/orchestrator-usage.md b/docs/orchestrator-usage.md index 8432884a..eca22d8e 100644 --- a/docs/orchestrator-usage.md +++ b/docs/orchestrator-usage.md @@ -1,5 +1,8 @@ # Orchestrator Usage — How-to Recipes +!!! abstract "Pillar context" + This page details **Pillar 1 — Plan with foresight** (and **Pillar 3 — Right agent, right problem, right time**). For the high-level map of all four pillars, see [The Four Pillars](pillars.md). + Practical, copy-pasteable recipes for driving Agent Baton. Each section answers a single "how do I X?" question. Commands link to [cli-reference.md](cli-reference.md) for full flag detail. diff --git a/docs/pillars.md b/docs/pillars.md new file mode 100644 index 00000000..7b104087 --- /dev/null +++ b/docs/pillars.md @@ -0,0 +1,105 @@ +# The Four Pillars + +Agent Baton is a **project manager for Claude Code**. Everything it does serves +four goals, in priority order. This page is the high-level map — what Baton is +*for* and how the parts fit together. Each pillar links to the deeper docs that +implement it. + +## What an effort looks like + +A task enters as plain language and leaves as tested, reviewed, auditable code. +Between those two points it passes through the four pillars in order. + +```mermaid +flowchart LR + T(["Your effort,
in plain language"]) --> P1 + P1["1 · Plan with foresight
decompose · risk · cost forecast"] --> P2 + P2["2 · Compose the right team
bespoke specialists · no context rot"] --> P3 + P3["3 · Right agent, right time
dispatch each phase · QA gates"] --> P4 + P4["4 · Checks & balances
independent auditor / SME verify"] --> D(["Tested, reviewed,
auditable code"]) +``` + +Governance is **Pillar 4** — a trust mechanism that keeps the project management +honest — not the headline. The pillars are the goals; a deep toolkit (CLI, REST +API, memory, observability) supports them and the people who run Baton day to +day. + +## Pillar 1 — Plan with foresight + +Before a single token goes to an agent, Baton builds a complete picture of the +work: what kind of task it is, which phases it needs, which agents fit them, +where it is likely to break, and what it will cost. You commit to a plan, not a +hope. `baton plan --dry-run` previews the phases, the agent assigned to each +step, the gates that will block, and a cost forecast (with an explicit ±50% +band) — all before execution. + +**→ [Read the full pillar page](pillars/plan-with-foresight.md)** — the vision, what ships today, and the gap. + +**Dive deeper:** [Orchestrator Usage](orchestrator-usage.md) · +[Engine & Runtime](engine-and-runtime.md) · +[State Machine](architecture/state-machine.md) + +## Pillar 2 — Compose the right team + +A single generalist agent on a complex codebase suffers **context rot** — by the +time it reaches step four it is carrying the weight of every earlier step plus +the whole codebase. Baton's answer is bespoke team composition: the +`talent-builder` creates narrowly-scoped specialists for *this* problem, so each +agent gets a clean context window focused on one domain. Routing then picks the +right stack-flavored variant for your project. + +**→ [Read the full pillar page](pillars/compose-the-right-team.md)** — the vision, what ships today, and the gap. + +**Dive deeper:** [Agent Roster](agent-roster.md) + +## Pillar 3 — Right agent, right problem, right time + +A deterministic engine dispatches each specialist to its assigned phase, gates +the output before advancing, and recovers from crashes without losing state. The +PMO flow organizes the whole effort from spec to merged PR. You drive it through +the `baton execute` command group; the engine does the sequencing. + +**→ [Read the full pillar page](pillars/right-agent-right-time.md)** — the vision, what ships today, and the gap. + +**Dive deeper:** [Engine & Runtime](engine-and-runtime.md) · +[Storage, Sync, & PMO](storage-sync-and-pmo.md) + +## Pillar 4 — Checks & balances + +Governance serves the pillars above it: it makes sure the right agent was +actually *right*, not just fast or syntactically correct. Risk is classified +before the first agent fires; medium/high-risk work pulls in an independent +`auditor` (with veto authority) and, for regulated domains, a +`subject-matter-expert`. Policy hooks enforce guardrails on every tool call, and +verifiable evidence bundles make the outcome auditable. + +**→ [Read the full pillar page](pillars/checks-and-balances.md)** — the vision, what ships today, and the gap. + +**Dive deeper:** [Knowledge, Events, & Governance](governance-knowledge-and-events.md) + +## The supporting layer + +The pillars are the goals. Around them sits a deep toolkit that supports both the +work and the people who run Baton: + +- **Reference for developers & contributors** — [CLI Reference](cli-reference.md), + [API Reference](api-reference.md), [Agent Roster](agent-roster.md), + [Terminology](terminology.md), [Invariants](invariants.md). +- **Memory, observability & learning** — + [Observe, Learn, & Improve](observe-learn-and-improve.md), + [Storage, Sync, & PMO](storage-sync-and-pmo.md). +- **Operations** — [Production Readiness](PRODUCTION_READINESS.md), + [Troubleshooting](troubleshooting.md), + [FinOps & Chargeback](finops-chargeback.md). + +## How the docs map to the pillars + +| Pillar | What it does | Full page | +|--------|--------------|-----------| +| **1 · Plan with foresight** | Decompose, sequence, classify risk, forecast cost before execution | [Plan with Foresight](pillars/plan-with-foresight.md) | +| **2 · Compose the right team** | Create bespoke specialists; route to the right flavor | [Compose the Right Team](pillars/compose-the-right-team.md) | +| **3 · Right agent, right time** | Deterministic dispatch, gates, crash recovery, PMO flow | [Right Agent, Right Time](pillars/right-agent-right-time.md) | +| **4 · Checks & balances** | Risk classification, auditor/SME verification, policy hooks, evidence | [Checks & Balances](pillars/checks-and-balances.md) | + +For the full conceptual background, start with the +[Architecture Overview](architecture.md). diff --git a/docs/pillars/checks-and-balances.md b/docs/pillars/checks-and-balances.md new file mode 100644 index 00000000..f8ce68c8 --- /dev/null +++ b/docs/pillars/checks-and-balances.md @@ -0,0 +1,345 @@ +--- +quadrant: explanation +audience: users, maintainers +see-also: + - [../pillars.md](../pillars.md) + - [../governance-knowledge-and-events.md](../governance-knowledge-and-events.md) +--- + +# Pillar 4 — Checks & Balances + +!!! abstract "Pillar context" + One of [the four pillars](../pillars.md) — the trust mechanism that keeps the project management honest. + +> **In one line:** verify the work is *functionally* right, not just that it lints — and make the outcome auditable. + +--- + +## The vision + +Fast and syntactically valid is not the same as correct. A task that passes +linting and unit tests can still violate a business rule, misinterpret a +regulatory requirement, or silently alter a schema in a way that breaks a +downstream compliance report. + +Pillar 4 addresses this by layering four kinds of checks: + +1. **Domain-expert verification.** The `auditor` agent is independent of the + orchestrator — it can overrule the plan. For regulated domains the + `subject-matter-expert` provides the domain context (regulatory requirements, + data models, validation rules) that the auditor and implementers need to be + right, not just done. Neither agent writes code; both can block code from + shipping. + +2. **Guardrails on every action.** Policy rules enforce constraints at the + tool-call level, not just at the plan level. Every `Write`, `Edit`, `Bash`, + and `MultiEdit` call is checked before it executes and recorded after. + +3. **Tamper-evident audit evidence.** The compliance audit log is hash-chained. + Evidence bundles package per-task artifacts under a SHA-256 manifest. Neither + can be silently altered after the fact. + +4. **Front-loading risk.** Spec federation imports externally-sourced work for + human review before any agent fires. Classification runs at plan time, not as + an afterthought. Catching a HIGH/CRITICAL risk before execution costs one API + call; catching it after costs a full rollback. + +What this is NOT: the verification does not improve itself autonomously or learn +to check better over time. That capability was cut as nascent and unproven. +Verification is done by human-authored agents operating on explicit domain +knowledge, policy rules that humans write, and deterministic hash functions. + +--- + +## How it works today + +### Risk classification + +**Module:** `agent_baton/core/govern/classifier.py` — `DataClassifier` + +Every task is classified before a single specialist fires. The classifier +scans the task description and affected file paths across five signal +categories, producing a `ClassificationResult` with a risk tier and the +matching guardrail preset: + +| Tier | Trigger | What fires | +|------|---------|-----------| +| LOW | No signals detected | Preset applied inline; no subagent overhead | +| MEDIUM | Database signals (migration, schema, alter table, …) | `auditor` reviews the plan before execution | +| HIGH | Regulated, PII, security, or infrastructure signals; or sensitive file paths (`.env`, `secrets/`, `auth/`, `terraform/`, …) | Independent `auditor` subagent with VETO authority; regulated domains also require `subject-matter-expert` | +| CRITICAL | Three or more regulated/PII signals in a single task (auto-escalation) | Same as HIGH | + +Risk can only be escalated, never lowered by a secondary signal. The +cascade is deterministic: regulated and PII signals dominate security +and infrastructure, which dominate database signals. + +When `ANTHROPIC_API_KEY` is set and the `agent-baton[classify]` extra is +installed, the planner uses an AI model for classification. Without it, the +classifier falls back to the keyword heuristic implemented in `classifier.py`. +Both paths produce the same `ClassificationResult` shape; the AI path produces +higher-quality signal on ambiguous task descriptions. + +```bash +baton classify "Add HIPAA audit trail to patient records" +baton classify "Refactor utility functions" --files src/auth/login.py +baton classify "Add HIPAA audit trail" --activate # also writes .claude/active-policy.json +``` + +### Independent auditor with veto authority + +**Agent:** `agents/auditor.md` + +The `auditor` agent runs in a separate context from the orchestrator by +design — it must be able to overrule the plan without being biased by the +planner's reasoning. It operates in three modes: + +1. **Pre-execution plan review.** Checks scope boundaries, write overlaps, + data safety, regulatory requirements, and rollback paths. Returns a + guardrails report with a per-agent permission manifest the orchestrator + enforces. +2. **Mid-execution checkpoints.** Returns CONTINUE / PAUSE / HALT at defined + step boundaries. A HALT prevents the next dependent step from dispatching. +3. **Post-execution audit.** Diff review, compliance scan, security scan, + domain validation. Returns a machine-readable verdict: + + | Verdict | Effect | + |---------|--------| + | `APPROVE` | Execution advances | + | `APPROVE_WITH_CONCERNS` | Advances; concerns are tracked | + | `REQUEST_CHANGES` | Revisions required before advancing | + | `VETO` | Halts HIGH/CRITICAL phase advancement | + +A `VETO` verdict blocks the executor from advancing HIGH/CRITICAL risk phases. +Overriding requires `--force --justification`, and every override is written to +the compliance audit chain — it cannot be silently discarded. + +The `AuditorVerdict` enum and its `blocks_execution` property are implemented +in `agent_baton/core/govern/compliance.py`. + +### Subject-matter expert for regulated domains + +**Agent:** `agents/subject-matter-expert.md` + +For any task that touches compliance systems, regulated data, or +industry-specific business rules (HIPAA, GDPR, SOX, PCI-DSS, FERPA, …), the +`subject-matter-expert` is required by the Regulated Data guardrail preset. +The SME supplies the domain context — regulatory constraints, data retention +rules, validation requirements, audit trail obligations — that implementers and +the auditor need to be correct. It does not write code; it provides the +knowledge that makes code correct. + +Regulated-domain rule from `agent_baton/core/govern/policy.py`: +the `regulated` preset requires both `subject-matter-expert` (`require_agent`, +severity `block`) and `auditor` (`require_agent`, severity `block`) in the +execution plan. Bash access on regulated data is also blocked by rule. + +### Policy hooks: enforcement on every tool call + +**Module:** `agent_baton/core/govern/policy.py` — `PolicyEngine` + +**Hook configuration:** `templates/settings.json` + +Two Claude Code hooks run on every tool call during execution: + +- **`baton policy-check`** (PreToolUse on `Bash|Write|Edit|MultiEdit`) — + evaluates the tool call against the active guardrail preset. A blocking + rule (`path_block`, `tool_restrict`) causes exit code 2, denying the + tool call before it executes. `BATON_POLICY_FAIL_CLOSED=1` makes hook + errors also deny; the default is fail-open. + +- **`baton comply-record`** (PostToolUse on `Bash|Write|Edit|MultiEdit`, + and on `Stop`) — appends a hash-chained entry to `compliance-audit.jsonl` + after each tool use. `BATON_COMPLIANCE_FAIL_CLOSED=1` makes write failures + halt execution rather than log-and-continue; required for regulated work + where losing an audit entry is itself a compliance defect. + +A separate inline path-block hook (also PreToolUse on `Write|Edit`) blocks +writes to `.env`, `secrets/`, `node_modules/`, and `.pem`/`.key` files at +the shell level before the policy engine even runs. + +The active preset is written to `.claude/active-policy.json` by +`baton classify --activate` or automatically by the execution engine when +it starts a task. The PreToolUse hook reads this file on every call; no +restart is required when the preset changes. + +Five built-in presets ship: `standard_dev`, `data_analysis`, `infrastructure`, +`regulated`, `security`. Custom presets live as JSON under `.claude/policies/`. + +### Assurance packs + +**Module:** `agent_baton/core/govern/packs.py` + +Organisations author domain-specific governance units under +`.claude/packs//`. Each pack bundles a policy set, classification +signals, a review rubric, gate definitions, and evidence requirements into +a single versioned directory: + +``` +.claude/packs// +├── pack.json # Manifest: name, version, description (required) +├── policy.json # PolicySet — preset name must be "pack:" +├── signals.json # Classification signals — keywords + path patterns +├── rubric.md # Review checklist (must have headings and checkboxes) +├── gates.json # Gate definitions (id, description, command) +└── evidence.json # Required artifacts (id, description) +``` + +Loading a pack merges its keyword signals into the `DataClassifier` and +registers its policy set so `policy-check` resolves `"pack:"` presets +automatically. When multiple packs match, the highest risk tier wins; ties +break alphabetically by preset name. + +```bash +baton packs init # Scaffold a new pack directory +baton packs validate # Validate all 7 schema checks +baton packs list # List loaded packs and their status +``` + +### Verifiable evidence bundles + +**Module:** `agent_baton/core/govern/evidence_bundle.py` — `EvidenceBundleBuilder`, `verify_bundle` + +After each task, `baton evidence bundle ` assembles a +self-contained directory under `evidence//`: + +| File | Contents | +|------|----------| +| `manifest.json` | SHA-256 digest of every other file in the bundle | +| `aibom.json` / `aibom.md` | AI Bill of Materials for the task | +| `compliance-segment.jsonl` | Task-scoped entries from the compliance audit chain | +| `gates.json` | Full gate results dump | +| `verdicts.json` | Auditor and reviewer step verdicts | +| `approvals.json` | Approval decisions and any pending approval request | +| `packs.json` | Active assurance packs + active-policy snapshot | + +`verify_bundle` checks every SHA-256 digest in `manifest.json`, verifies the +internal consistency of the compliance segment's hash chain, and (when +`--sign` was used) verifies the soul signature on the manifest. It accepts +either a directory or a `.tar.gz` archive and is network-free, suitable for +CI. + +```bash +baton evidence bundle # Build bundle +baton evidence bundle --tar # Build and compress to .tar.gz +baton evidence bundle --sign # Sign manifest (requires BATON_SOULS_ENABLED=1) +baton evidence verify # Verify directory or .tar.gz; exits 0/1/2 +``` + +### Segregation-of-duties approval + +`BATON_APPROVAL_MODE=team` requires the approving actor to differ from whoever +requested the approval. Self-approval is blocked at the engine level. The +approval request records the requester identity; the approval result records +the actor. This satisfies the basic segregation-of-duties requirement for +regulated work. + +### Spec federation: the cheapest control point + +Before any agent fires on externally-sourced work, a spec can be imported from +GitHub Issues or Azure DevOps, auto-enriched with a risk classification and +cost forecast (pack-aware when packs are loaded), and routed for senior review. +A HIGH/CRITICAL spec can be bounced at this stage for one API call rather than +discovering the problem mid-execution. + +```bash +baton spec import # Import from GitHub Issues / Azure DevOps +baton spec list +baton spec approve # Blocked on self-approval in team mode +baton spec bounce # Return with feedback +``` + +The `SpecDraftStore` backing these routes is in +`agent_baton/api/routes/spec_queue.py`. + +--- + +## The gap today + +The checks-and-balances layer is functional, but several parts are experimental +or have documented limits. + +**Persistent agent souls are experimental.** `BATON_SOULS_ENABLED` defaults to +`0`. When disabled, evidence bundles are built and SHA-256 verified +correctly, but `manifest.json` is unsigned and `verdicts.json` carries no +soul-signature fields. Cryptographic attribution of who produced and signed +each verdict is only available with souls enabled. + +**The `soul.verify()` bypass (bd-1ca2).** `Soul.verify()` is a pure +cryptographic check — it does not consult the revocation registry. This means +any caller that calls `soul.verify()` directly instead of going through +`SoulRouter.verify_signature()` will accept a revoked soul's signature as +valid. The regression tests in +`tests/test_soul_verify_revocation_through_callers.py` document and cover this +bug. `SoulRouter.verify_signature()` is the correct call site; callers that +have not yet been migrated to it are not protected by the revocation guard. +Evidence bundle signing routes through the correct path via +`agent_baton/core/govern/evidence_bundle.py`, but the bd-1ca2 issue means +any future caller that reaches for `soul.verify()` directly will silently +bypass revocation enforcement. + +**Evidence bundle signing depends on the experimental souls feature.** The +`--sign` flag on `baton evidence bundle` is gated behind +`BATON_SOULS_ENABLED=1`. When souls are disabled, `--sign` emits a warning +and produces an unsigned bundle. Tamper detection via SHA-256 manifest +verification still works without souls; cryptographic signing of the manifest +and verdict attribution do not. + +**The executable-beads sandbox is process-level only.** When +`BATON_EXEC_BEADS_ENABLED=1`, scripts stored as executable beads run inside a +sandbox in `agent_baton/core/exec/` that enforces a wall-clock timeout, +memory limit, captured stdout/stderr, a static lint denylist, and an +operator-confirmation prompt plus auditor gate. It does NOT provide filesystem +namespacing, network namespacing, or a syscall filter. The trust model assumes +scripts are locally-authored, version-controlled, and team-reviewed. Scripts +from external origins — federation, downloaded packs, fork PRs, customer +uploads — are not covered by this sandbox. `baton beads exec` emits a +`[security]` warning when it detects a non-local `source` value, but that +warning is a tripwire, not a defence. (Source: `docs/architecture.md`, +"Trust Boundary" section.) + +**Keyword-only classification when `ANTHROPIC_API_KEY` is absent.** The +`DataClassifier` in `agent_baton/core/govern/classifier.py` is a keyword +matching engine by default. AI-powered classification (higher accuracy on +ambiguous task descriptions) requires `BATON_API_KEY` set and the +`agent-baton[classify]` extra installed. Deployments without the API key fall +back silently to the keyword heuristic; tasks with unusual phrasing may be +under-classified. + +**Policy hooks run via Claude Code hooks, not the executor.** `baton +policy-check` and `baton comply-record` are Claude Code settings hooks, not +in-process enforcement inside the Python engine. This means policy evaluation +depends on Claude Code loading `settings.json` and on the hooks being invoked +correctly. Hooks that fail (e.g., because the `baton` binary is not on PATH) +are fail-open by default (`BATON_POLICY_FAIL_CLOSED=0`). There is no +executor-level backstop that catches a policy rule violation the hook missed. + +--- + +## Where this lives + +| Area | Location | +|------|----------| +| Governance explanation | [../governance-knowledge-and-events.md](../governance-knowledge-and-events.md) | +| Risk classifier | `agent_baton/core/govern/classifier.py` | +| Policy engine | `agent_baton/core/govern/policy.py` | +| Compliance chain | `agent_baton/core/govern/compliance.py` | +| Assurance packs | `agent_baton/core/govern/packs.py` | +| Evidence bundles | `agent_baton/core/govern/evidence_bundle.py` | +| Auditor agent | `agents/auditor.md` | +| Subject-matter-expert agent | `agents/subject-matter-expert.md` | +| Hook configuration | `templates/settings.json` | +| Spec federation routes | `agent_baton/api/routes/spec_queue.py` | + +**Commands:** + +```bash +baton classify "" # Risk classification +baton classify "" --activate # Classification + activate preset +baton policy # List presets +baton policy --show regulated # Show rules in a preset +baton evidence bundle # Build evidence bundle +baton evidence verify # Verify bundle integrity +baton packs list # List assurance packs +baton packs validate # Validate pack structure +baton compliance # List compliance reports +``` diff --git a/docs/pillars/compose-the-right-team.md b/docs/pillars/compose-the-right-team.md new file mode 100644 index 00000000..c2101f3f --- /dev/null +++ b/docs/pillars/compose-the-right-team.md @@ -0,0 +1,282 @@ +--- +quadrant: explanation +audience: users, maintainers +see-also: + - [../pillars.md](../pillars.md) + - [../agent-roster.md](../agent-roster.md) +--- + +# Pillar 2 — Compose the Right Team + +!!! abstract "Pillar context" + One of [the four pillars](../pillars.md) — the differentiator: bespoke specialists over overloaded generalists. + +> **In one line:** the right specialists for *this* problem, each with a clean, focused context window. + +--- + +## The vision + +The ideal is a system that auto-composes a purpose-built fleet for every task: +zero context rot, a specialist for every domain, no one agent dragging the +weight of earlier phases into a new one. + +A single generalist agent on a complex codebase suffers **context rot**: by the +time it reaches step four it is carrying the full history of the previous three +steps plus however much of the codebase it was given on entry. Reasoning quality +degrades, token cost compounds, and errors from earlier steps bleed forward +unchecked. + +The answer is narrowly-scoped specialists. Instead of one agent asked to plan, +implement backend logic, write tests, and review security in a single session, +Baton assembles an ad-hoc fleet where each member: + +- Receives a **single, bounded task** — one domain, one phase. +- Starts from a **clean context window** loaded only with the knowledge + relevant to that task. +- Is **purpose-built for its stack**: a Python-FastAPI project gets + `backend-engineer--python`, not a generic backend agent that must infer + Python idioms from scratch. + +When a gap in the roster is discovered — no specialist exists for the +combination of role and domain needed — `talent-builder` fills it: it researches +the domain, creates the agent file, builds a knowledge pack, and optionally +scaffolds a repeatable skill, so the next time the same gap arises the fleet is +already equipped. + +--- + +## How it works today + +### The 30 shipping agents + +`scripts/install.sh` installs **30 agent definitions** from `agents/` into +`.claude/agents/` (project scope) or `~/.claude/agents/` (user scope). The +`agents/CLAUDE.md` confirms the count; `baton agents` shows them at runtime, +grouped by category. + +The roster covers the full delivery lifecycle: + +| Category | Agents | +|----------|--------| +| Orchestration | `orchestrator`, `team-lead`, `task-runner` | +| Backend | `backend-engineer`, `backend-engineer--python`, `backend-engineer--node` | +| Frontend | `frontend-engineer`, `frontend-engineer--react`, `frontend-engineer--dotnet` | +| Architecture | `architect` | +| Quality | `test-engineer`, `code-reviewer`, `security-reviewer` | +| Governance | `auditor` | +| Data | `data-engineer`, `data-analyst`, `data-scientist` | +| Visualization | `visualization-expert` | +| Operations | `devops-engineer` | +| Domain | `subject-matter-expert`, `learning-analyst`, `system-maintainer` | +| Meta | `talent-builder` | +| Archetype | `archetype-james-engineering-manager` | +| Resilience | `immune-autofix`, `immune-deprecated-api`, `immune-doc-drift`, `immune-stale-comment`, `immune-todo-rot`, `immune-untested-edges` | + +Each file is a Markdown document with YAML frontmatter (`name`, `description`, +`model`, `tools`, optional `permissionMode`, `color`) and a body system prompt. +The frontmatter is what the runtime registers; the body is what the agent reads +when dispatched. See `agents/CLAUDE.md` for format rules. + +### The `role--flavor` naming scheme + +Stack-specific variants follow the convention `--`. Four flavors +ship today: + +| Flavored agent | Use when | +|----------------|----------| +| `backend-engineer--python` | `pyproject.toml`, `requirements.txt`, or `setup.py` at root | +| `backend-engineer--node` | `package.json` or `tsconfig.json` at root (JS/TS project) | +| `frontend-engineer--react` | `next.config.*`, `nuxt.config.*`, `angular.json`, or `vite.config.*` with `"react"` in `package.json` | +| `frontend-engineer--dotnet` | `appsettings.json` or `.csproj`/`.sln` at root | + +The base agents (`backend-engineer`, `frontend-engineer`) remain in the registry +as fallbacks when no matching flavor exists. + +### Routing: stack detection and flavor selection + +`baton route [ROLES]` and `baton agents` are the CLI entry points +(`agent_baton/cli/commands/agents/route.py`, `agents.py`). + +The `AgentRouter` in `agent_baton/core/orchestration/router.py` runs a two-pass +scan of the project tree (root + visible children + visible grandchildren, +skipping `node_modules`, `__pycache__`, `dist`, `build`, `.git`): + +1. **Framework signals** (more specific) — `FRAMEWORK_SIGNALS` maps filenames + like `next.config.js` or `appsettings.json` to `(language, framework)` pairs. + Root-level signals are authoritative; subdir-level signals provide framework + hints but do not override a root-level language. + +2. **Package manager signals** (broader) — `PACKAGE_SIGNALS` maps `pyproject.toml`, + `go.mod`, `Cargo.toml`, etc. to a language. Python wins over Node/TS when both + appear at the root (monorepo convention: Python backend + JS frontend). + +The result is a `StackProfile(language, framework, detected_files, languages, +frameworks)`. The router then consults `FLAVOR_MAP` to find the right flavor +suffix for each requested role, verifies the flavored agent exists in the +`AgentRegistry`, and returns either the flavored name or the base name as +fallback. + +```bash +baton route backend-engineer frontend-engineer +# Stack: python/fastapi +# backend-engineer → backend-engineer--python * +# frontend-engineer → frontend-engineer (no flavor match) +``` + +### Learned overrides + +Routing corrections that persist across sessions are stored in +`.claude/team-context/learned-overrides.json` and managed by +`LearnedOverrides` in `agent_baton/core/learn/overrides.py`. Before consulting +the hardcoded `FLAVOR_MAP`, the router reads `flavor_map` from this file. A +project-specific entry wins: + +```json +{ + "flavor_map": { + "python/react": { + "backend-engineer": "python", + "frontend-engineer": "react" + } + } +} +``` + +The `system-maintainer` agent is the designated writer of +`learned-overrides.json` — it never touches source code. + +### The `talent-builder` agent + +When the roster has a gap, `talent-builder` (`agents/talent-builder.md`, runs on +`opus`) fills it. It builds the full knowledge stack, not just an agent file: + +| Artifact | Location | When created | +|----------|----------|--------------| +| Agent definition (`.md`) | `.claude/agents/` or `~/.claude/agents/` | A new role is needed | +| Knowledge pack | `.claude/knowledge//` | Domain facts too large to bake into the prompt (100–500 lines) | +| Skill | `.claude/skills//SKILL.md` + scripts + templates | A workflow is done repeatedly | +| Reference doc | `.claude/references/` | Multiple agents share the same knowledge | + +`talent-builder` follows a structured workflow: understand the need, research +the domain (light or deep, 5–30 minutes), apply a five-test decision framework +(agent vs. knowledge pack vs. skill vs. reference doc), and report a token-cost +estimate before writing anything. The naming convention it uses matches the +fleet's existing scheme: `backend-engineer--go`, `data-analyst--salesforce`, etc. + +### Context economics + +Every subagent costs a full context-window load, startup latency, and +information-loss at handoff. Baton manages this cost at four points: + +1. **Inline research and routing** — stack detection and knowledge resolution + run inside the orchestrator's own session; no new agent window is opened + until actual implementation work begins (noted in `README.md` and + `agent_baton/core/orchestration/CLAUDE.md`). + +2. **Per-step MCP pass-through** — each `PlanStep` declares `mcp_servers`: + only the listed servers are forwarded into the agent's tool environment. + Steps that need no external tools carry an empty list, keeping unused tool + schemas out of the context window (source: `README.md` §"Selective MCP + pass-through"; `docs/design-decisions.md` §ADR-21). + +3. **CHECKPOINT action** — `ActionType.CHECKPOINT` (`agent_baton/models/ + execution.py`, value `"checkpoint"`) tells the orchestrator to save state + and start a fresh session. Its docstring reads: *"save state + suggest fresh + session to prevent context rot"*. State persists to `baton.db`; `baton + execute resume` reconstructs from the last checkpoint without re-dispatching + any step. + +4. **Worktree isolation** — when two or more steps in a wave are parallel-safe, + the executor provisions a linked git worktree at `.claude/worktrees/ + //` before dispatch. Parallel agents write to isolated + working copies and no uncommitted change leaks between steps + (`BATON_WORKTREE_ENABLED`, default `1`; stale worktrees reclaimed after + 4 hours by `WorktreeManager.gc_stale()`). + +--- + +## The gap today + +### 1. Limited shipping flavor coverage + +Four flavors ship (`--python`, `--node`, `--react`, `--dotnet`). The stack +detector knows how to detect Go (`go.mod`), Rust (`Cargo.toml`), Ruby +(`Gemfile`), Java (`build.gradle`, `pom.xml`), and Kotlin — but none of these +languages appear in `FLAVOR_MAP`, so the router always falls back to the +unflavored `backend-engineer` for Go, Rust, Ruby, and Java projects. The +detection data is there; the specialist agent is not. + +Source: `agent_baton/core/orchestration/router.py` — `PACKAGE_SIGNALS` lists +all six non-covered languages; `FLAVOR_MAP` has no entry for any of them. + +### 2. On-demand creation friction + +When a gap exists, `talent-builder` can fill it — but it does not do so +automatically mid-plan. It is a dispatched agent step, not a background service. +The flow is: planner emits a DISPATCH action → orchestrator spawns +`talent-builder` → `talent-builder` conducts its research interview (Step 1 of +its workflow asks for domain, documentation, scope, and intended usage) → writes +the files → returns. This adds at least one full agent round-trip before the +missing specialist is available, and the orchestrator must then restart or +reconfigure the affected steps. + +There is no auto-initiation path that detects a missing flavor at `baton plan` +time and automatically invokes `talent-builder` to fill the gap before execution +begins (the `has_project_agents()` check in `AgentRegistry` triggers a +talent-builder suggestion in `baton plan`, but it is advisory, not automatic). + +### 3. Heuristic routing and mis-routes + +The router is deterministic for a given `(task, registry)` pair — `orchestration/ +CLAUDE.md` calls this out explicitly as a design goal. However, the signals it +uses are purely file-system heuristics. Two known failure modes: + +- **Root vs. subdir priority**: a Python monorepo with `pyproject.toml` at the + root and a React app in `pmo-ui/` correctly reports `language=python`. But a + project with only a subdir-level framework signal (e.g., `pmo-ui/next.config.js` + in an otherwise empty parent) triggers the subdir-framework fallback path, + which can classify the whole repo as TypeScript/React. This is explicitly + called out in the router's inline comments (bd-75e8 fix). + +- **Language-not-in-FLAVOR_MAP**: Go, Rust, Ruby, Java, and Kotlin are detected + but produce no flavor. The router silently falls back to the base agent and + logs at `DEBUG` level — no warning surfaces to the operator that the detected + stack is uncovered. + +`learned-overrides.json` is the correction mechanism: once a mis-route is +identified, `system-maintainer` writes the correct mapping, and subsequent runs +use it. But the initial mis-route still happens once before the correction +exists. + +### 4. Context rot mitigated, not eliminated + +CHECKPOINT, worktrees, and per-step MCP pass-through reduce context +accumulation, but they do not eliminate it: + +- **CHECKPOINT** suggests a fresh session; it does not force one. Whether a + new session is started depends on the orchestrator agent acting on the + instruction. +- **Worktree isolation** prevents uncommitted file bleed between parallel + agents but does not reduce context within a single long-running specialist + session. +- **Knowledge packs** front-load domain knowledge as structured files instead + of re-deriving it from the codebase, but the agent still loads and processes + them, contributing to context size. +- **Per-step MCP pass-through** keeps tool schemas small, not the + accumulated conversation history. + +There is no mechanism to measure context pressure within a running specialist +session and proactively checkpoint before quality degrades. The CHECKPOINT +action is inserted at plan time based on step count or phase boundaries, not +dynamically based on observed context size. + +--- + +## Where this lives + +- Docs: [../agent-roster.md](../agent-roster.md), [../orchestrator-usage.md](../orchestrator-usage.md) +- Code: `agents/talent-builder.md`, `agent_baton/core/orchestration/router.py`, + `agent_baton/core/orchestration/registry.py`, + `agent_baton/core/learn/overrides.py` +- Commands: `baton agents`, `baton route` diff --git a/docs/pillars/plan-with-foresight.md b/docs/pillars/plan-with-foresight.md new file mode 100644 index 00000000..2d928928 --- /dev/null +++ b/docs/pillars/plan-with-foresight.md @@ -0,0 +1,137 @@ +--- +quadrant: explanation +audience: users, maintainers +see-also: + - [../pillars.md](../pillars.md) + - [../engine-and-runtime.md](../engine-and-runtime.md) +--- + +# Pillar 1 — Plan with Foresight + +!!! abstract "Pillar context" + One of [the four pillars](../pillars.md) — the first thing Baton does for every effort. + +> **In one line:** see the shape of the work, and where it will break, before spending a token. + +## The vision + +Before a single specialist agent fires, you should be able to hold the entire effort in one view: what kind of task this is, which phases it needs in which order, which agents fit each phase, where the work is likely to break, and what it will cost — stated with explicit uncertainty, not false precision. You commit to a plan backed by analysis, not a hope backed by optimism. + +The ideal classifier takes a plain-language description, detects the project stack, and produces a task type, complexity tier, and a ranked agent roster with high confidence — surfacing signals the author may not have articulated ("this touches migrations, so you need rollback provisioned first"). Risk flows from content, not from human memory: regulated keywords, sensitive file paths, and cross-domain integration signals each elevate the tier automatically, so the guardrail preset and approval requirements are set before any code is written. + +Cost and wall-clock time should be knowable up front — not a surprise invoice after the run. The forecast should be honest about its own precision: a ±50% band is more useful than a false decimal place. And when a first plan does not fully satisfy a stated goal, the engine should be able to identify the gap, append the missing phases, and re-evaluate — converging on "done" rather than stopping at "shipped what I planned." + +Finally, the plan itself should be transparent. Every decision — why this agent, why HIGH risk, why this phase sequence — should be explicable on demand, so a human can review and correct it before the first agent runs. + +## How it works today + +### The seven-stage planning pipeline + +`baton plan ""` calls `IntelligentPlanner.create_plan()`, which runs a seven-stage deterministic pipeline defined in `agent_baton/core/engine/planning/pipeline.py`. The stages run in fixed order: + +1. **ClassificationStage** — generates a task ID, auto-detects the project stack via `AgentRouter.detect_stack()`, infers task type (new-feature, bug-fix, migration, refactor, test, audit, …), and assigns a complexity tier (light / medium / heavy). The primary path uses `FallbackClassifier` in `agent_baton/core/engine/classifier.py`, which tries `TalentAgentClassifier` (Sonnet via Claude CLI) first and falls back to `KeywordClassifier` (deterministic keyword heuristics) when the CLI is unavailable. +2. **RosterStage** — selects the agent roster for this task type and complexity, drawing on the agent registry, prior retrospective patterns, and stack routing to pick stack-flavored variants (e.g. `backend-engineer--python`). +3. **RiskStage** — runs `DataClassifier.classify()` from `agent_baton/core/govern/classifier.py` to assign a risk level (LOW / MEDIUM / HIGH / CRITICAL) and a guardrail preset. The classifier matches the task description and changed file paths against five signal categories (regulated, PII, security, infrastructure, database). Three or more regulated/PII signals escalate the risk to CRITICAL automatically. Assurance Packs can extend the classifier with domain-specific keywords and path patterns via `make_classifier_for_packs()`. +4. **DecompositionStage** — builds phases from templates keyed on task type and complexity (`agent_baton/core/engine/planning/rules/phase_templates.py`). After phases are built, the **Foresight Engine** (`agent_baton/core/engine/foresight.py`) runs as a sub-step: it scans every step description and agent assignment against seven built-in rules and inserts preparatory phases when a gap is detected (see below). +5. **EnrichmentStage** — attaches gate commands to each phase boundary, derives approval requirements from the risk tier, injects knowledge packs and reference documents, and surfaces prior bead hints from completed executions. +6. **ValidationStage** — scores the plan quality, assigns a budget tier (tight / standard / generous), and runs the structural quality gate (see `agent_baton/core/engine/planning/stages/validation.py`). Critical defects (empty plan, empty phase, wrong agent role in a phase, missing Review or Audit coverage) raise `PlanQualityError` and block by default. `BATON_DEV_MODE=1` or `BATON_PLANNER_WARN_ONLY=1` downgrade those defects to warnings for local experimentation; `BATON_PLANNER_HARD_GATE=1` forces blocking even in those modes. +7. **AssemblyStage** — assembles the final `MachinePlan` and emits OpenTelemetry spans if `BATON_OTEL_ENABLED` is set. + +### The Foresight Engine + +`ForesightEngine` in `agent_baton/core/engine/foresight.py` is the engine that inserts preparatory phases you did not explicitly ask for but that are necessary for success. It runs as part of DecompositionStage (step 9.7 in the legacy numbering, before shared context is assembled). + +The engine maintains seven built-in rules, each with a `rule_id`, trigger keywords, trigger agents, a confidence score, and a `resolution_template`. When a rule matches a step description and the assigned agent, the engine inserts a new phase before the triggering phase. Examples: + +- **`foresight-migration-rollback`** (confidence 0.9) — any step mentioning "migrate", "alter table", or "drop column" triggers insertion of a "Prepare: Migration Safety" phase that sets up reversible scripts and pre-migration backups before the migration runs. +- **`foresight-api-schema`** (confidence 0.8) — API endpoint steps trigger an architect-led "Prepare: API Schema" phase to define request/response schemas before implementation begins. +- **`foresight-destructive-safety`** (confidence 0.85) — steps mentioning "delete", "drop", "truncate", or "purge" trigger a safety-check phase adding dry-run mode and audit logging. +- **`foresight-integration-contract`** (confidence 0.75) — cross-domain integration steps trigger an architect-led contract-definition phase. + +Higher-risk plans lower the confidence threshold (from 0.7 to 0.5 for HIGH/CRITICAL), so more rules fire when the stakes are higher. Duplicate insertions for the same rule are collapsed into a single preparatory phase. + +### CLI surface and key flags + +```bash +# Preview the plan and cost forecast without saving: +baton plan "add OAuth2 login" --dry-run + +# Generate, save, and explain the rationale: +baton plan "add OAuth2 login" --save --explain + +# Override complexity if the classifier gets it wrong: +baton plan "move one config key" --complexity light --save + +# Plan against a completion condition; engine amends until met: +baton goal "all four integration tests pass" --max-amend-cycles 3 + +# Import a hand-crafted plan instead of auto-generating: +baton plan --import my-plan.json --save + +# Surgically fix a saved plan without regenerating: +baton plan-edit --swap-agent 1.1 backend-engineer--python +baton plan-edit --set-risk HIGH +baton plan-edit --add-phase Review --add-agent code-reviewer +``` + +`--dry-run` renders a compact preview: phases, steps, assigned agents, gates that will block, and a cost forecast with an explicit ±50% confidence band. It exits without writing anything. `--explain` writes a human-readable rationale file to `.claude/team-context/explanation.md` covering pattern influence, risk signals, foresight insertions, and agent routing notes. Both flags are mutually exclusive with `--save`. + +### Cost forecasting + +`agent_baton/core/engine/cost_estimator.py` sums per-step token allowances using role-specific baselines (architect/code-reviewer: 8 000 tokens; backend-engineer/frontend-engineer/test-engineer: 5 000; everything else: 4 000) and multiplies by a blended I/O price per model family. The output always includes the ±50% band — the comment in `plan_cmd.py` at line 359 references `bd-47b4` as the decision to surface this band explicitly so "developers do not treat the dollar figure as authoritative." + +### Optional LLM plan-quality review + +After the deterministic pipeline, an optional LLM pass is available via `BATON_PLAN_REVIEW=haiku|sonnet|opus`. When enabled, the plan is sent to the selected model for structural quality review (step splitting, dependency gaps, scope balance). The result can add parallel steps, team steps, or dependency edges. The feature is off by default; `sonnet` is the recommended setting for unattended planning. + +### Goal-driven amendment loop + +`baton goal ""` stamps a `completion_condition` on the plan and a `max_amend_cycles` budget (default: 3). After each gate passes, `ExecutionEngine._evaluate_goal_after_gate` calls the `GoalEvaluator`. If the goal is not yet met and the amend budget remains, `amend_plan` appends new phases and execution continues. If the budget is exhausted, the engine emits `FAILED` with reason `"goal not met, amend budget exhausted"`. The `BATON_GOAL_EVALUATOR` variable selects the evaluation strategy: `stub` (deterministic, no LLM), `haiku`, or `opus`. + +## The gap today + +### 1. Classifier accuracy depends on API key availability + +The primary classification path uses `TalentAgentClassifier` (Sonnet via the `claude` CLI), which brings broad language understanding to task-type, complexity, and agent-roster decisions. When `ANTHROPIC_API_KEY` is unset or the `claude` CLI is unavailable, `FallbackClassifier` silently degrades to `KeywordClassifier`, a deterministic keyword-scoring implementation in `agent_baton/core/engine/classifier.py`. + +`KeywordClassifier` works well for clearly phrased, single-domain tasks but has known failure modes: ambiguous task descriptions that require reading intent rather than matching keywords, tasks that describe the mechanism rather than the goal ("update the config value" instead of "rename the env var"), and tasks that span multiple types simultaneously. The fallback path logs a warning but does not block — the user may not realize they received a lower-quality plan. + +**What would close this gap:** make the fallback explicit in `--dry-run` output and surface it as a plan defect in `ValidationStage`. Consider expanding `KeywordClassifier`'s test coverage to characterize its known blind spots. + +### 2. Complexity assessment has known limits + +The CLAUDE.md for this repo states directly: "The deterministic pipeline has known limits in complexity assessment." `KeywordClassifier` uses regex signals (`_LIGHT_QUANTIFIERS`, `_HEAVY_SCOPE`, `_HEAVY_ARCH`) and gives no weight to project-specific context: a task that is "light" in a greenfield project may be "heavy" in a deeply coupled legacy codebase. The planner has no mechanism to read existing code complexity or dependency graphs before assigning the tier. + +Compensating controls exist: the default planner quality gate blocks structurally defective plans (empty phases, role mismatches, missing Review or Audit coverage), and the spec queue provides pre-flight human review. The `--complexity` flag is an explicit escape hatch. But neither control addresses the root cause — the classifier does not know what "complex" means for your codebase. + +**What would close this gap:** inject a static analysis signal (cyclomatic complexity, file coupling metrics) as a hint to the complexity stage, or route complexity assessment through the LLM classifier unconditionally when the API key is available. + +### 3. Cost forecast is a coarse ±50% estimate + +The cost estimator uses fixed per-role baselines and does not account for: knowledge pack size (which dominates token usage on document-heavy tasks), retries triggered by gate failures, INTERACT phases that can span many turns, or actual model pricing changes. The `bd-47b4` note in the source code acknowledges this directly — the ±50% band is stated on every forecast output precisely because the estimate is not reliable enough to omit it. + +The wall-clock estimate adds agent minutes and gate minutes using heuristic constants (`estimate_gate_seconds`) that do not vary by project environment. + +**What would close this gap:** calibrate the baseline estimates against retrospective actuals (the `observe` + `learn` subsystem already collects per-step token usage); feed that history back into `cost_estimator.py` so the bands tighten over time. + +### 4. Foresight rules are limited to seven built-in patterns + +`ForesightEngine` ships with seven rules covering data CRUD completeness, migration rollback safety, API schema validation, destructive operation safety, infrastructure environment preparation, integration contract definition, and test infrastructure scaffolding. These rules fire on keyword matching and agent-name matching — there is no semantic understanding of what a step actually does. A step titled "handle user records" that internally drops columns will not trigger `foresight-migration-rollback` unless the keywords appear in the step description. + +There is no mechanism to author custom foresight rules per project. Plans that touch domains outside the seven built-in categories (e.g. embedded systems, custom DSLs, regulated clinical workflows) will not receive foresight insertions. + +**What would close this gap:** expose a foresight rule extension point in assurance packs (analogous to how packs already extend the risk classifier's keyword lists); provide a `baton plan --explain` section that lists which foresight rules fired and which did not, so gaps in coverage are visible. + +### 5. `BATON_PLAN_REVIEW` is off by default + +The optional LLM post-pipeline review can catch structural quality issues — overly broad single-step phases, missing dependency edges, scope imbalance — that the deterministic pipeline cannot assess. But it is disabled by default because it adds latency and API cost. This means most plans in the wild ship without the review, including plans generated in CI pipelines or managed-mode automation where human review is unlikely. + +The CLAUDE.md table notes: "The deterministic pipeline has known limits in complexity assessment; default compensating controls are the structural hard gate and pre-flight human review in the spec queue — enable this for unattended/managed-mode planning." + +**What would close this gap:** run the LLM review automatically when `BATON_GOAL_EVALUATOR` is already set (i.e. the user has opted into LLM-backed evaluation), or default to `haiku` review for `heavy` complexity plans where the deterministic pipeline's limits are most likely to matter. + +## Where this lives + +- Docs: [../engine-and-runtime.md](../engine-and-runtime.md), [../architecture/state-machine.md](../architecture/state-machine.md), [../orchestrator-usage.md](../orchestrator-usage.md) +- Code: `agent_baton/core/engine/planning/` (full pipeline), `agent_baton/core/engine/foresight.py` (Foresight Engine), `agent_baton/core/engine/cost_estimator.py` (cost forecast), `agent_baton/core/engine/classifier.py` (task classifier + fallback), `agent_baton/core/govern/classifier.py` (risk/data classifier), `agent_baton/core/engine/plan_reviewer.py` (structural reviewer) +- Commands: `baton plan --dry-run`, `baton plan --explain`, `baton plan --save`, `baton plan-edit`, `baton goal` diff --git a/docs/pillars/right-agent-right-time.md b/docs/pillars/right-agent-right-time.md new file mode 100644 index 00000000..b5c75b67 --- /dev/null +++ b/docs/pillars/right-agent-right-time.md @@ -0,0 +1,197 @@ +--- +quadrant: explanation +audience: users, maintainers +see-also: + - [../pillars.md](../pillars.md) + - [../engine-and-runtime.md](../engine-and-runtime.md) + - [../storage-sync-and-pmo.md](../storage-sync-and-pmo.md) +--- + +# Pillar 3 — Right Agent, Right Problem, Right Time + +!!! abstract "Pillar context" + One of [the four pillars](../pillars.md) — the deterministic engine that sequences the work. + +> **In one line:** each phase to the specialist that fits it, gated and resumable, from spec to merge. + +--- + +## The vision + +The ideal expressed by Pillar 3 is a dispatch loop that is **fully deterministic, fully resumable, and impossible to lose work in**. + +- A plan arrives as a `MachinePlan` — a graph of phases, steps, dependencies, and gates. +- The engine walks that graph exactly, dispatching each step to the named specialist and nothing else. +- Between phases an automated QA gate fires. The next phase does not start until it passes. +- Human approval gates pause the loop and wait for a signed decision before advancing. +- If the process crashes at any point, `baton execute resume` picks up from the last persisted state — no steps repeat, no steps are lost. +- Parallel steps with disjoint file scopes run under separate git worktrees so they cannot silently clobber each other. +- The loop runs unattended in daemon mode, and the PMO UI organizes the whole effort from spec import through merged PR. + +The long-form expression of that ideal: spec-to-merge with zero information loss, zero duplicated work after a crash, and zero specialist overlap — each agent gets a clean context window focused on exactly one problem. + +--- + +## How it works today + +### The state machine and 9 action types + +The execution engine (`agent_baton/core/engine/executor.py`) is a **synchronous, stateless state machine**. Every call reads `ExecutionState` from disk, computes the next action, writes updated state atomically (write to `.json.tmp`, rename), and returns. This means a crash between any two calls loses nothing. + +The engine returns one of 9 `ActionType` values defined in `agent_baton/models/execution.py`: + +| Action | When returned | +|--------|--------------| +| `DISPATCH` | The next step is ready; spawn the named agent with the provided prompt | +| `GATE` | All steps in this phase are complete; run the QA gate command | +| `APPROVAL` | Phase requires human sign-off before advancing | +| `FEEDBACK` | Present multiple-choice questions; the chosen answer dispatches a follow-up step | +| `INTERACT` | Agent asked a clarifying question; pause for human reply, then re-dispatch | +| `WAIT` | Parallel steps are still in flight; call `next_action()` again | +| `COMPLETE` | All phases exhausted; execution is finished | +| `FAILED` | A step or gate failed unrecoverably | +| `CHECKPOINT` | Engine suggests saving state and opening a fresh session to prevent context rot | + +The driving session (a Claude Code orchestrator or the headless `TaskWorker`) loops over these actions, records results with `baton execute record` and `baton execute gate`, and advances until `COMPLETE` or `FAILED`. + +### The dispatch loop in practice + +``` +baton plan "task" --save --explain # writes plan.json + plan.md +baton execute start # initialises ExecutionState, returns first action +baton execute run # headless loop: dispatches all steps autonomously +baton execute complete # writes trace, usage log, retrospective +``` + +For interactive use, the orchestrator agent drives the loop manually: `baton execute start` → agent reads the action, spawns the specialist, calls `baton execute record` → `baton execute next` → repeat until `COMPLETE`. Full recipe in `docs/orchestrator-usage.md` and `references/baton-engine.md`. + +### Crash recovery + +State is persisted after every `record_*` call. If the process dies between calls, `baton execute resume` loads the last good state and returns the next action exactly as if nothing happened. Steps already marked `complete` are skipped; steps marked `dispatched` (in-flight at crash time) are recovered by `recover_dispatched_steps()`, which clears the stale marker so the engine re-dispatches them. + +Resume also restores the cumulative spend counter from `ExecutionState.run_cumulative_spend_usd` so the token ceiling continues counting from the right baseline rather than resetting to zero. + +### Concurrency: `BATON_TASK_ID` and parallel execution + +Multiple tasks can run simultaneously. `BATON_TASK_ID` (or `--task-id`) targets a specific execution when several are in flight. Task-ID resolution order: `--task-id` flag → `BATON_TASK_ID` env var → `active-task-id.txt` → error. + +Within a task, steps whose `depends_on` sets are satisfied can run in parallel. `engine.next_actions()` returns all currently dispatchable steps in one call. The async `TaskWorker` (`agent_baton/core/runtime/worker.py`) collects this batch, marks each step dispatched, and runs them through `StepScheduler` (`agent_baton/core/runtime/scheduler.py`) — an `asyncio.Semaphore`-bounded pool capped at `max_concurrent` (default 3). + +### Worktree isolation + +When steps run concurrently, `WorktreeManager` (`agent_baton/core/engine/worktree_manager.py`) creates a separate git worktree for each step under `.claude/worktrees///`. The agent process runs with that worktree as its working directory. On success the worktree is folded back into the parent branch; on failure it is preserved for forensic inspection or developer takeover. Stale worktrees are reclaimed by `gc_stale()` after 72 hours (`BATON_WORKTREE_GC_HOURS`). + +The `_WORKTREE_DISCIPLINE_BLOCK` injected into every isolation-mode prompt tells the agent exactly how to operate inside its worktree boundary. + +### Team steps and synthesis strategies + +A `PlanStep` with a non-empty `team` list is a team step. Each `TeamMember` carries a `member_id` (e.g. `"1.1.a"`), a `role` (lead/implementer/reviewer), and optional intra-step `depends_on` links. + +When all members complete, outputs are merged via `SynthesisSpec`: + +| Strategy | Behaviour | +|----------|-----------| +| `concatenate` (default) | Join outcomes with `"; "`; collect all files changed | +| `merge_files` | Same but deduplicate `files_changed` | +| `agent_synthesis` | Dispatch a synthesis agent to merge outputs (see gap below) | + +Conflict handling when two members modify the same file is controlled by `SynthesisSpec.conflict_handling`: `auto_merge` (default — record in retrospective and complete), `escalate` (surface to human via `APPROVAL`), or `fail`. + +Two team backends are available and both are supported (selected via `BATON_TEAMS_BACKEND`): + +- **`worktree`** (default): parallel `Agent` calls under git worktree isolation; resumable; full agent frontmatter honored. +- **`claude-teams`** (opt-in): native Agent Teams UX with inter-teammate messaging and shared task list; requires `CLAUDE_CODE_EXPERIMENTAL_AGENT_TEAMS=1`. + +### Selective MCP pass-through + +Steps declare which MCP servers they need via `PlanStep.mcp_servers`. Only declared servers are passed to the agent subprocess via `--mcp-config`. Undeclared servers are excluded, preventing input-token bloat from tool schemas the agent will never call. + +### Plan amendments mid-flight + +If a gate fails or an approver returns feedback, `baton execute amend` inserts new phases or steps into the live plan without stopping execution. Every amendment writes a `PlanAmendment` audit record to `ExecutionState.amendments`. Goal-driven execution (`baton goal ""`, enabled via `MachinePlan.completion_condition`) automatically runs `amend_plan()` at phase boundaries when the `GoalEvaluator` determines the completion condition has not been met, up to `max_amend_cycles` (default 3) times. + +### Headless execution and daemon mode + +`baton execute run` uses `HeadlessClaude` (`agent_baton/core/runtime/headless.py`) to drive the full dispatch loop without a Claude Code session — each dispatch calls `claude --print` as a subprocess. This is the mode used by the PMO execute endpoint and by CI pipelines. + +`baton daemon` daemonizes the `WorkerSupervisor` (`agent_baton/core/runtime/supervisor.py`) via UNIX double-fork (`agent_baton/core/runtime/daemon.py`). The supervisor manages the PID file, rotating log, and graceful shutdown on SIGTERM/SIGINT (30-second drain window). It is not available on Windows. + +A separate `ImmuneDaemon` (`agent_baton/core/immune/daemon.py`) runs background anti-rot sweeps when `BATON_IMMUNE_ENABLED=1`. It ticks every 5 minutes, picking sweep targets from a SQLite queue and routing findings through `FindingTriage`. Its state is resumable after a crash. + +### PMO plan-to-merge flow + +The PMO REST API (`agent_baton/api/routes/spec_queue.py`) supports a structured plan-to-merge pipeline: + +1. Import a spec from GitHub Issues or Azure DevOps, or submit one directly. +2. Enrich it with `DataClassifier` cost forecasting. +3. Senior review: approve or bounce. +4. Fire: trigger headless execution. +5. Monitor: the PMO board tracks per-step and per-phase status in real time. +6. Merge: the `CommitConsolidator` cherry-picks agent commits onto the feature branch; the changelist and attribution are visible in the UI. + +--- + +## The gap today + +Three honest gaps between the vision above and the current implementation. + +### Gap 1 — Token ceiling warns but does not block individual dispatches (bd-3f80) + +`BATON_RUN_TOKEN_CEILING` is a USD ceiling for the cumulative spend of the run. The `BudgetEnforcer` (`agent_baton/core/govern/budget.py`) raises `RunTokenCeilingExceeded` before any **immune-sweep LLM call** that would push cumulative spend past the ceiling. + +However, `Executor.dispatch` does **not** call `enforce_run_ceiling()` before firing individual agent dispatches. The ceiling is enforced only through the policy-hook path — a `baton policy-check` hook evaluated at each tool call. If no such hook is wired up, the engine logs a warning at HIGH/CRITICAL risk start (`warn_if_ceiling_unset_for_high_risk`) and appends `TOKEN_BUDGET_WARNING` to step deviations after the fact, but it does not block a DISPATCH action that would exceed the ceiling. + +**Practical effect**: on unattended HIGH/CRITICAL runs without a `BATON_POLICY_FAIL_CLOSED=1` policy hook, the ceiling is advisory, not a hard kill. Set `BATON_POLICY_FAIL_CLOSED=1` and wire `baton policy-check` into `PreToolUse` hooks to get hard enforcement. + +### Gap 2 — `claude-teams` backend cannot resume in-flight teammates + +The `worktree` backend (default) is fully resumable. The `claude-teams` backend is not: `baton execute resume` can reload `ExecutionState` and continue the enclosing plan, but it cannot revive Claude-Teams teammates that were in-flight when the session died. The native Agent Teams protocol has no in-process resumption mechanism. + +Additional constraints of the `claude-teams` backend (documented in `agent_baton/core/engine/team_backends.py` and `docs/engine-and-runtime.md` §18): no nested teams, `skills` and `mcpServers` frontmatter on teammate definitions is not honored, one team at a time per lead session, and approximately 7x the token overhead of the worktree path. + +`BATON_TEAMS_STRICT_RESUMABILITY=1` causes `baton plan` / `baton goal` to refuse to save a plan with team phases if the claude-teams backend is active and the budget tier is `long-running`. Default (`0`) downgrades to a warning. + +### Gap 3 — `agent_synthesis` team strategy is declared but not yet dispatched + +`SynthesisSpec.strategy = "agent_synthesis"` is the intended path for having a dedicated synthesis agent merge the outputs of multiple team members into a coherent whole. The enum value, the `synthesis_agent` field, and the `synthesis_prompt` template are all defined in `agent_baton/models/execution.py`. In the current executor (`agent_baton/core/engine/executor.py`), however, the synthesis agent dispatch is not wired: team step auto-completion uses only `concatenate`/`merge_files` semantics. The `agent_synthesis` value is safe to set in a plan but behaves identically to `concatenate` until the dispatch path is implemented. + +--- + +## Where this lives + +**Docs** + +- `../engine-and-runtime.md` — full reference: state machine, planner, dispatcher, gate system, runtime, crash recovery, team backends +- `../storage-sync-and-pmo.md` — PMO plan-to-merge flow and spec queue +- `../architecture/state-machine.md` — every action type, every status value, every persistence touchpoint + +**Code** + +- `agent_baton/core/engine/executor.py` — `ExecutionEngine` (state machine, dispatch loop, budget checks) +- `agent_baton/core/engine/dispatcher.py` — `PromptDispatcher` (delegation prompts, worktree discipline block) +- `agent_baton/core/engine/gates.py` — `GateRunner` +- `agent_baton/core/engine/persistence.py` — `StatePersistence` (atomic writes, task-ID resolution) +- `agent_baton/core/engine/worktree_manager.py` — `WorktreeManager` +- `agent_baton/core/engine/team_backends.py` — `WorktreeTeamBackend`, `ClaudeTeamsBackend` +- `agent_baton/core/runtime/worker.py` — `TaskWorker` (async execution loop) +- `agent_baton/core/runtime/scheduler.py` — `StepScheduler` (bounded-concurrency dispatch) +- `agent_baton/core/runtime/headless.py` — `HeadlessClaude` (`baton execute run`) +- `agent_baton/core/runtime/daemon.py` — `daemonize()` (double-fork) +- `agent_baton/core/runtime/supervisor.py` — `WorkerSupervisor` (PID file, graceful shutdown) +- `agent_baton/core/immune/daemon.py` — `ImmuneDaemon` (background anti-rot sweeps) +- `agent_baton/core/govern/budget.py` — `BudgetEnforcer`, `RunTokenCeilingExceeded` +- `agent_baton/models/execution.py` — `ActionType` (9 values), `MachinePlan`, `ExecutionState`, `SynthesisSpec` + +**Commands** + +```bash +baton plan "" --save --explain # create plan.json + plan.md +baton execute start # initialise state, return first action +baton execute next # return next action (interactive loop) +baton execute record --step-id ... # record step result +baton execute gate --phase-id --result pass|fail +baton execute resume # crash recovery +baton execute run # headless autonomous loop +baton daemon start # daemonised autonomous execution +baton daemon status # list running daemon workers +``` diff --git a/docs/storage-sync-and-pmo.md b/docs/storage-sync-and-pmo.md index a82460a7..8e5aa2f0 100644 --- a/docs/storage-sync-and-pmo.md +++ b/docs/storage-sync-and-pmo.md @@ -5,6 +5,9 @@ SQLite databases, the central read-replica, federated sync, external source adapters, cross-project queries, the PMO portfolio overlay, and the PMO UI frontend. +!!! abstract "Pillar context" + This page covers **Pillar 3 — Right agent, right problem, right time**: the PMO plan-to-merge flow, the persistence layer that stores and replays every execution, and the cross-project analytics that keep the right work moving through the right hands at the right moment. For the high-level map of all four pillars, see [The Four Pillars](pillars.md). + --- ## 1. Overview diff --git a/docs/terminology.md b/docs/terminology.md index cc8640cb..136010b8 100644 --- a/docs/terminology.md +++ b/docs/terminology.md @@ -15,6 +15,7 @@ Canonical terms used across the codebase, CLI, and docs. Alphabetical. | **Action** | A unit emitted by the engine to drive the orchestration loop. One of: DISPATCH, GATE, APPROVAL, COMPLETE, FAILED, WAIT, FEEDBACK, INTERACT. See `ActionType` in `agent_baton/models/execution.py`. | | **Agent** | A distributable specialist defined in `agents/.md` (frontmatter + prompt body). Dispatched by the orchestrator or invoked directly via Claude Code's `Agent` tool. | | **Approval** | Explicit human (or designated reviewer) sign-off required for HIGH-risk plans and certain phase transitions. Driven by the APPROVAL action and `baton execute approve`. | +| **Assurance pack** | A governance bundle in `.claude/packs//` with a `pack.json` manifest, policy/signals/rubric files, and optional gates/evidence. Distinct from knowledge packs: assurance packs enforce checks and risk controls. | | **Bead** | A persistent incident or follow-up record stored by the external `bd` tool ([gastownhall/beads](https://github.com/gastownhall/beads)) in a per-project `.beads/` workspace. Created with `baton beads create`. Used for autonomous bug filing, regression beads, audit trails. | | **Baton** | The project name (capitalized in prose). | | **`baton`** | The CLI binary (lowercase, monospace). | diff --git a/mkdocs.yml b/mkdocs.yml index 49adbd82..f38215fb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: Agent Baton -site_description: Multi-agent orchestration system for Claude Code — intelligent planning, delegation, and governance for complex software engineering tasks. +site_description: Agent Baton is a project manager for Claude Code — plan with foresight, compose the right specialist team, dispatch the right agent at the right time, and keep work on track with checks and balances. site_url: https://davegerson.github.io/agent-baton/ site_author: Dave Gerson repo_url: https://github.com/DaveGerson/agent-baton @@ -79,7 +79,11 @@ markdown_extensions: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.snippets - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tabbed: alternate_style: true - pymdownx.tasklist: @@ -105,6 +109,12 @@ extra: nav: - Home: index.md + - The Four Pillars: + - pillars.md + - Plan with Foresight: pillars/plan-with-foresight.md + - Compose the Right Team: pillars/compose-the-right-team.md + - Right Agent, Right Time: pillars/right-agent-right-time.md + - Checks & Balances: pillars/checks-and-balances.md - Getting Started: - Orchestrator Usage: orchestrator-usage.md - Agent Roster: agent-roster.md diff --git a/reference_docs/framing_and_roadmap/00-roadmap-index.md b/reference_docs/framing_and_roadmap/00-roadmap-index.md new file mode 100644 index 00000000..bb874059 --- /dev/null +++ b/reference_docs/framing_and_roadmap/00-roadmap-index.md @@ -0,0 +1,78 @@ +# Agent Baton Short-Term Capability Roadmaps + +**Purpose:** Four phased roadmaps that Baton can use to implement quick and short-term wins for Agent Baton without structural refactoring. These plans focus on end-user capabilities that improve software outcomes for developers using the tool: better plans, safer coordination, clearer team execution, reusable agent/knowledge assets, and more predictable day-to-day operation. + +**Operating constraint:** Do not split major modules, redesign storage, replace the planning pipeline, or perform broad API/UI rewrites. Each task should be implementable as a targeted feature, validation improvement, test, CLI/API addition, or documentation improvement. + +--- + +## How to use these roadmaps with Baton + +Each capability file has four phases. To run a phase, copy the relevant **Baton run prompt** into your normal Agent Baton workflow. Recommended execution pattern: + +```text +Use Agent Baton to implement Phase from .md. +Follow the constraints, acceptance criteria, and validation commands exactly. +Do not perform structural refactoring outside the file/path scope listed in the roadmap. +``` + +Run the capabilities in the order below when possible: + +1. Plan creation and coordination +2. Knowledge pack management +3. Talent Builder and subagent management +4. Agent team spin-up +5. General developer experience, packaging, and layout polish + +This order wires dormant capabilities first, then improves validation, then improves generated assets and teams, then makes the result easier to install and operate. + +--- + +## Four-phase delivery model + +| Phase | Theme | End-user outcome | Definition of done | +|---|---|---|---| +| **Phase 1** | Make hidden capability visible and active | Developers can see what Baton is doing, why it chose a plan/team/knowledge set, and whether core capability wiring is active. | Capability is enabled by default or reports clearly when unavailable. CLI/API outputs include actionable diagnostics. Focused tests cover the new behavior. | +| **Phase 2** | Make outputs trustworthy | Developers get fewer malformed plans, bad dispatches, missing context, and silent degradations. | Validation failures are actionable. Important quality gates are enforced or explicitly opt-out. Golden/smoke tests cover common workflows. | +| **Phase 3** | Make workflows reusable | Developers can create, review, and reuse agents, teams, and knowledge packs with fewer manual checks. | Doctor/validate commands catch missing references, unsafe permissions, stale knowledge, and weak team contracts. | +| **Phase 4** | Make it shippable day-to-day | Developers can install, inspect, and operate Baton consistently across projects. | Docs, CLI help, packaging, UI visibility, and release checks support the improved workflows. | + +--- + +## Capability matrix + +| Capability | Phase 1 | Phase 2 | Phase 3 | Phase 4 | +|---|---|---|---|---| +| **Plan creation and coordination** | Wire planner defaults, plan diagnostics, explainability summary | Plan hard-gate defaults, golden plan tests, clearer defects | Context/handoff quality, goal/round-out visibility | PMO plan preview, docs, planner smoke pack | +| **Agent team spin-up** | Team readiness audit, backend strictness, team report | File-scope contracts, conflict severity, synthesis minimum viable path | Team validation and mailbox visibility | PMO team status and team playbook docs | +| **Talent Builder / subagent management** | Canonical naming, generated-agent contract, validation checklist | Agent doctor, knowledge reference verification, permission warnings | Draft/review/promote workflow using metadata | Agent catalog docs/UI and starter templates | +| **Knowledge pack management** | Wire KnowledgeRegistry by default, manifest normalization | `baton knowledge doctor/search`, pack validation, body-index option | Gap-to-pack suggestions, freshness/usage signals | Knowledge dashboard/docs and example packs | +| **General layout / developer UX** | Doctor command, terminology cleanup, package-resource audit | CI smoke matrix, import/package tests, CLI help snapshots | PMO auth/client polish, install verification | Release checklist, documentation nav, examples | + +--- + +## Global non-goals + +These roadmaps deliberately exclude structural refactoring. Do **not** use these plans to: + +- split `ExecutionEngine` into services, +- split `api/routes/pmo.py`, +- replace SQLite/file persistence, +- redesign the PMO UI, +- change the public `MachinePlan` schema without migration handling, +- replace the planning pipeline, +- introduce a new agent runtime. + +Those are valid medium-term workstreams, but they are outside this quick/short-term roadmap. + +--- + +## Global quality bar + +Every phase should include: + +- at least one focused unit test or integration smoke test, +- CLI/API output that helps a developer understand what changed, +- no broad module moves, +- no silent fallback where the user needs a clear warning, +- updated docs or help text when behavior changes. diff --git a/reference_docs/framing_and_roadmap/01-plan-creation-and-coordination.md b/reference_docs/framing_and_roadmap/01-plan-creation-and-coordination.md new file mode 100644 index 00000000..3d07f3ee --- /dev/null +++ b/reference_docs/framing_and_roadmap/01-plan-creation-and-coordination.md @@ -0,0 +1,242 @@ +# Roadmap: Plan Creation and Coordination + +**Capability goal:** Developers should get plans that are well-scoped, explainable, context-aware, and safe to execute. Baton should show why it selected phases, agents, gates, knowledge, and coordination constraints. + +**No-structural-refactor constraint:** Keep the existing `IntelligentPlanner` pipeline, `PlanDraft`, `ExecutionEngine`, `ActionResolver`, and PMO route structure. Add wiring, diagnostics, validation, tests, and small behavior improvements only. + +--- + +## Phase 1 — Activate and expose planning intelligence + +### Developer outcome + +A developer running `baton plan` or using PMO Forge can see what Baton inferred: task type, complexity, archetype, risk, selected agents, selected phases, knowledge attachments, gates, and validation warnings. + +### Work items + +1. **Wire the default knowledge registry into planner construction.** + - Build and load `KnowledgeRegistry` in CLI/API planner entry points when no explicit registry is provided. + - If no packs exist, report `knowledge_registry: loaded=0`, not an error. + - If packs are degraded, report degraded pack names in diagnostics. + +2. **Add a concise plan diagnostics block.** + - Include task type, complexity, archetype, risk, classification source, selected agents, phase count, gate count, approval count, knowledge attachment count, and validation warning count. + - Prefer a stable text block plus optional JSON output. + +3. **Expose planner explainability in normal workflows.** + - Add or extend `--explain` / `--explain-json` support for plan creation. + - PMO Forge should return explainability metadata or make it queryable after plan generation. + +4. **Make unavailable dependencies visible.** + - If LLM classification, knowledge registry, bead store, or policy engine is unavailable, print a clear warning with the fallback path. + - Avoid noisy stack traces unless `--debug` is set. + +### Suggested files + +```text +agent_baton/core/engine/planning/planner.py +agent_baton/core/engine/planning/stages/assembly.py +agent_baton/core/orchestration/knowledge_registry.py +agent_baton/cli/commands/**/plan*.py +agent_baton/api/deps.py +agent_baton/api/routes/pmo.py +tests/planning/ +tests/api/ +``` + +### Acceptance criteria + +- A plan created in a project with `.claude/knowledge//knowledge.yaml` attaches or references matching knowledge documents. +- A plan created in a project with no knowledge packs still succeeds and says no knowledge packs were loaded. +- Plan output includes a short diagnostics summary. +- `explain_plan()` or equivalent output is available through CLI and PMO/API path. +- Tests cover both with-knowledge and no-knowledge cases. + +### Validation commands + +```bash +python -m pytest -q tests/planning tests/api +python -m pytest -q tests/test_api_pmo_beads.py || true +baton plan "Add a small validation helper and test it" --explain +``` + +### Baton run prompt + +```text +Implement Phase 1 of roadmaps/01-plan-creation-and-coordination.md. +Focus on activating KnowledgeRegistry by default and surfacing concise plan diagnostics for developers. +Do not refactor the planner pipeline or ExecutionEngine. +Add focused tests for knowledge-loaded and no-knowledge cases. +``` + +--- + +## Phase 2 — Make plan quality failures actionable + +### Developer outcome + +Developers should not receive malformed or low-quality plans without clear warnings. When Baton detects a bad plan, it should explain exactly what to fix. + +### Work items + +1. **Make critical plan defects fail by default in non-dev mode.** + - Preserve an explicit opt-out for local experimentation. + - Keep `BATON_PLANNER_HARD_GATE` support, but add a clearer default policy such as `BATON_DEV_MODE=1` to allow warnings-only behavior. + +2. **Improve defect messages.** + - For `empty_plan`, `empty_phase`, `agent_phase_mismatch`, and `review_skipped`, include phase/step IDs and a suggested remediation. + +3. **Add golden plan tests.** + - Snapshot representative plans: + - direct/light task, + - investigative bug task, + - compound multi-concern task, + - high-risk/security task, + - compliance/audit task, + - knowledge-heavy task. + +4. **Fail clearly on impossible phase/agent assignment.** + - If reviewer/auditor agents are blocked from implementation phases and no Review/Audit phase is created, return a plan defect. + +### Suggested files + +```text +agent_baton/core/engine/planning/stages/validation.py +agent_baton/core/engine/planning/utils/phase_builder.py +agent_baton/core/engine/planning/stages/enrichment.py +tests/planning/test_plan_quality_*.py +tests/snapshots/plans/ +``` + +### Acceptance criteria + +- Critical defects block plan creation unless dev/warn-only mode is explicitly enabled. +- Every critical defect includes a human-readable remediation. +- Golden plan tests are stable and intentionally updated when planner behavior changes. +- High-risk and compliance plans always contain review/audit coverage or fail validation. + +### Validation commands + +```bash +python -m pytest -q tests/planning/test_plan_quality_*.py +python -m pytest -q tests/snapshots || true +BATON_PLANNER_HARD_GATE=1 baton plan "Refactor authentication and payment authorization logic" +``` + +### Baton run prompt + +```text +Implement Phase 2 of roadmaps/01-plan-creation-and-coordination.md. +Make critical planner defects actionable and enforceable without restructuring the planner. +Add golden tests for representative plan shapes. +``` + +--- + +## Phase 3 — Improve context, handoffs, and coordination loops + +### Developer outcome + +Agents should receive the right context at the right time, with less repeated knowledge and clearer prior-step handoffs. Developers should see when Baton amends or rounds out a plan. + +### Work items + +1. **Add context budget reporting.** + - For each plan, report estimated shared context size, inline knowledge size, reference count, and largest context contributor. + +2. **Improve extracted path warnings.** + - If extracted file paths are outside the project root, mark them read-only in diagnostics and prompt context. + +3. **Make goal round-out visible.** + - When goal evaluation inserts phases, record a concise amendment summary in status output and plan history. + +4. **Add handoff quality checks.** + - Warn when a step depends on a previous step that produced no outcome, no files, and no bead/handoff summary. + +### Suggested files + +```text +agent_baton/core/engine/planning/utils/context.py +agent_baton/core/engine/dispatcher.py +agent_baton/core/engine/executor.py +agent_baton/cli/commands/execution/execute.py +agent_baton/api/routes/pmo.py +tests/engine/ +tests/planning/ +``` + +### Acceptance criteria + +- Plan diagnostics include context size estimates. +- Out-of-root paths are visible to the developer and are not presented as write targets. +- Status output shows plan amendments/goal round-out cycles. +- Tests cover out-of-root path rendering and empty-handoff warning behavior. + +### Validation commands + +```bash +python -m pytest -q tests/engine tests/planning +baton plan "Update src/api.py and review /tmp/example-only-reference.txt" --explain +``` + +### Baton run prompt + +```text +Implement Phase 3 of roadmaps/01-plan-creation-and-coordination.md. +Improve context and handoff visibility for developers without changing core state-machine structure. +Add focused tests for path warnings and context-size diagnostics. +``` + +--- + +## Phase 4 — Make planning inspectable in PMO and docs + +### Developer outcome + +A developer using the PMO UI can preview why Baton selected a plan and can reject/edit the plan before execution with enough context to make a good decision. + +### Work items + +1. **Add PMO plan preview metadata.** + - Show classification source, risk, agents, phases, gates, knowledge attachments, and validation warnings. + +2. **Add docs for plan interpretation.** + - Explain what each diagnostic field means and how to fix common warnings. + +3. **Add plan smoke examples.** + - Provide small example tasks and expected plan characteristics. + +4. **Add release check for planner smoke.** + - Include planner smoke tests in CI/release gating. + +### Suggested files + +```text +pmo-ui/src/** +agent_baton/api/models/responses.py +agent_baton/api/routes/pmo.py +docs/cli-reference.md +docs/orchestrator-usage.md +tests/api/ +pmo-ui/src/**/*.test.tsx +``` + +### Acceptance criteria + +- PMO preview displays plan diagnostics before approval/save. +- Docs include a troubleshooting table for plan warnings. +- CI has a planner smoke target or required job. + +### Validation commands + +```bash +python -m pytest -q tests/api tests/planning +cd pmo-ui && npm run build && npm run test:run +``` + +### Baton run prompt + +```text +Implement Phase 4 of roadmaps/01-plan-creation-and-coordination.md. +Expose plan diagnostics in PMO and documentation. Do not redesign the PMO UI; add a compact preview panel and focused tests. +``` diff --git a/reference_docs/framing_and_roadmap/02-agent-team-spinup.md b/reference_docs/framing_and_roadmap/02-agent-team-spinup.md new file mode 100644 index 00000000..105f07f8 --- /dev/null +++ b/reference_docs/framing_and_roadmap/02-agent-team-spinup.md @@ -0,0 +1,224 @@ +# Roadmap: Agent Team Spin-Up Capabilities + +**Capability goal:** Developers should be able to ask Baton for a multi-agent team and receive coordinated, bounded, reviewable work with clear ownership, conflict visibility, and synthesis. + +**No-structural-refactor constraint:** Keep the existing `WorktreeTeamBackend`, `ClaudeTeamsBackend`, `TeamMember`, `TeamStepResult`, mailbox, and execution state model. Add validation, diagnostics, prompt improvements, reports, and minimal synthesis behavior only. + +--- + +## Phase 1 — Make team readiness visible + +### Developer outcome + +Before Baton launches a team, developers should know whether the team is safe to run, which backend is active, which teammates will be spawned, and which limitations apply. + +### Work items + +1. **Add team readiness diagnostics.** + - For every team step, report backend, member count, nested team count, shared files/contracts, synthesis strategy, conflict strategy, and warning count. + +2. **Strict backend option.** + - Add `BATON_TEAMS_BACKEND_STRICT=1` or equivalent. + - In strict mode, unknown team backend names fail instead of falling back to worktree. + +3. **Surface Claude Teams caveats to users.** + - When `claude-teams` is active, print or return warnings for no resume, no nesting, one-team-at-a-time, fixed permissions, and missing skills/MCP frontmatter. + +4. **Add team dispatch report artifact.** + - Write a lightweight `team-report.md` or JSON under the execution/team directory when a team step is dispatched. + +### Suggested files + +```text +agent_baton/core/engine/team_backends.py +agent_baton/core/engine/planning/utils/phase_builder.py +agent_baton/core/engine/executor.py +agent_baton/cli/commands/execution/execute.py +tests/engine/test_team_*.py +``` + +### Acceptance criteria + +- Developers can see the active team backend before dispatch. +- Unknown backend fails in strict mode and falls back only in default permissive mode. +- A team report exists for each team step. +- Claude Teams warnings are present in CLI/API diagnostics. + +### Validation commands + +```bash +python -m pytest -q tests/engine/test_team_*.py +BATON_TEAMS_BACKEND=not-real BATON_TEAMS_BACKEND_STRICT=1 python -m pytest -q tests/engine/test_team_*.py +``` + +### Baton run prompt + +```text +Implement Phase 1 of roadmaps/02-agent-team-spinup.md. +Add team readiness diagnostics, strict backend behavior, and a per-team report artifact. +Do not redesign team execution or replace the existing backends. +``` + +--- + +## Phase 2 — Improve ownership and conflict quality + +### Developer outcome + +Team members should receive clearer file-scope contracts, and Baton should catch likely conflicts before they become confusing failures. + +### Work items + +1. **Add machine-readable file ownership contracts.** + - Extend team diagnostics/reporting with member-level intended file or path scope. + - Use existing prompt fields where possible; do not add a new runtime model unless necessary. + +2. **Warn on overlapping ownership before dispatch.** + - If two members have identical or overlapping `allowed_paths` / file-scope text, add a warning. + +3. **Add conflict severity.** + - Current conflict detection is file-overlap based. Keep it, but classify severity: + - high: same file modified by two implementers, + - medium: shared config/test files, + - low: docs or generated artifacts. + +4. **Improve conflict messages.** + - Include affected files, member IDs, agents, and next action. + +### Suggested files + +```text +agent_baton/core/engine/executor.py +agent_baton/core/engine/team_backends.py +agent_baton/core/engine/planning/utils/phase_builder.py +agent_baton/models/retrospective.py +tests/engine/test_team_conflicts.py +``` + +### Acceptance criteria + +- Team report shows per-member ownership contract. +- Overlap warnings appear before dispatch when ownership is ambiguous. +- Conflict records include severity and actionable details. +- Existing team tests continue to pass. + +### Validation commands + +```bash +python -m pytest -q tests/engine/test_team_conflicts.py tests/engine/test_team_*.py +``` + +### Baton run prompt + +```text +Implement Phase 2 of roadmaps/02-agent-team-spinup.md. +Improve team member ownership diagnostics and conflict severity using targeted changes only. +Avoid model/schema changes unless a small backward-compatible optional field is clearly necessary. +``` + +--- + +## Phase 3 — Add minimum viable team synthesis + +### Developer outcome + +After a team completes, developers should receive one coherent summary of what the team did, what changed, what remains risky, and whether follow-up review is required. + +### Work items + +1. **Improve existing synthesis strategies.** + - Keep `concatenate` and `merge_files`, but format output as a readable team summary instead of a semicolon string. + +2. **Implement a minimal `agent_synthesis` fallback.** + - If true agent dispatch is too much for this phase, create a deterministic synthesis report and mark that agent synthesis was requested but deterministic fallback was used. + - If small enough, dispatch the configured synthesis agent as a follow-up review step using existing dispatch mechanics. + +3. **Add a team summary artifact.** + - Store final team summary in the execution directory. + +4. **Add tests for synthesis output.** + - Cover concatenate, merge_files, and agent_synthesis fallback. + +### Suggested files + +```text +agent_baton/core/engine/executor.py +agent_baton/core/engine/dispatcher.py +tests/engine/test_team_synthesis.py +``` + +### Acceptance criteria + +- Team parent outcome is readable Markdown. +- Files changed are deduplicated when strategy requires it. +- Agent synthesis request is not silently ignored. +- Tests cover all synthesis strategies. + +### Validation commands + +```bash +python -m pytest -q tests/engine/test_team_synthesis.py tests/engine/test_team_*.py +``` + +### Baton run prompt + +```text +Implement Phase 3 of roadmaps/02-agent-team-spinup.md. +Add a minimum viable synthesis report for team steps and tests for all existing synthesis strategies. +Do not introduce a new team runtime. +``` + +--- + +## Phase 4 — Make teams inspectable in PMO and docs + +### Developer outcome + +Developers should be able to inspect team progress, teammate outputs, conflicts, synthesis, and warnings from the PMO UI or CLI without reading raw JSON state. + +### Work items + +1. **Expose team details in card/execution detail.** + - Include team members, member status, conflict status, synthesis summary, and team report path. + +2. **Add PMO team panel.** + - Simple read-only panel; no major redesign. + +3. **Document team backends.** + - Explain worktree vs Claude Teams tradeoffs, resumability, nesting, skills/MCP caveats, and strict mode. + +4. **Add team smoke test to CI.** + - At minimum, verify a team plan can be generated and team diagnostics can be rendered. + +### Suggested files + +```text +agent_baton/api/routes/pmo.py +agent_baton/api/models/responses.py +pmo-ui/src/** +docs/engine-and-runtime.md +docs/orchestrator-usage.md +tests/api/ +pmo-ui/src/**/*.test.tsx +``` + +### Acceptance criteria + +- PMO card execution detail displays team members and synthesis summary. +- CLI status exposes team report path. +- Docs clearly state which backend to choose for reliable/resumable execution. + +### Validation commands + +```bash +python -m pytest -q tests/api tests/engine/test_team_*.py +cd pmo-ui && npm run build && npm run test:run +``` + +### Baton run prompt + +```text +Implement Phase 4 of roadmaps/02-agent-team-spinup.md. +Expose team execution status in API/PMO and document backend tradeoffs. +Keep UI changes small and read-only. +``` diff --git a/reference_docs/framing_and_roadmap/03-talent-builder-subagent-management.md b/reference_docs/framing_and_roadmap/03-talent-builder-subagent-management.md new file mode 100644 index 00000000..ec393529 --- /dev/null +++ b/reference_docs/framing_and_roadmap/03-talent-builder-subagent-management.md @@ -0,0 +1,237 @@ +# Roadmap: Talent Builder and Subagent Management + +**Capability goal:** Developers should be able to create, validate, review, and reuse specialist subagents without accidentally creating unsafe, bloated, or broken agent definitions. + +**Canonical term:** The implemented agent is `talent-builder`. Treat `talent-manager` as an alias only if needed for user-facing compatibility. + +**No-structural-refactor constraint:** Keep filesystem-backed agent definitions and `AgentRegistry` override semantics. Add validation, metadata, templates, CLI helpers, and documentation only. + +--- + +## Phase 1 — Standardize generated-agent contract + +### Developer outcome + +When Talent Builder creates or updates an agent, developers receive predictable files, metadata, references, and validation instructions. + +### Work items + +1. **Add a generated-agent output contract.** + - Define required frontmatter fields: `name`, `description`, `model`, `permissionMode`, `tools`. + - Define recommended fields: `owner`, `status`, `version`, `created_by`, `last_reviewed`, `knowledge_packs`. + - Define output sections: mission, before starting, knowledge references, principles, anti-patterns, output format. + +2. **Update Talent Builder instructions.** + - Ensure it writes the contract into every generated agent. + - Add explicit “do not use broad tools unless needed” rule. + - Add “read back and validate references” as non-optional. + +3. **Add `talent-manager` alias documentation.** + - If docs or prompts use `talent-manager`, point to `talent-builder`. + - Avoid creating two divergent agents unless there is a strong reason. + +4. **Create starter templates.** + - `templates/agents/base-agent.md` + - `templates/agents/flavored-agent.md` + - `templates/agents/reviewer-agent.md` + +### Suggested files + +```text +agents/talent-builder.md +templates/agents/*.md +references/agent-authoring.md +docs/agent-roster.md +tests/agents/ +``` + +### Acceptance criteria + +- Talent Builder instructions include the generated-agent contract. +- Templates exist and match the contract. +- Docs use `talent-builder` consistently or clearly alias `talent-manager`. +- Existing bundled agents still parse. + +### Validation commands + +```bash +python -m agent_baton.cli.main validate agents +python -m pytest -q tests/agents || true +``` + +### Baton run prompt + +```text +Implement Phase 1 of roadmaps/03-talent-builder-subagent-management.md. +Standardize the generated-agent contract, update Talent Builder instructions, and add templates. +Do not change AgentRegistry architecture. +``` + +--- + +## Phase 2 — Add agent doctor validation + +### Developer outcome + +Developers can run one command to find broken generated agents before Baton dispatches them. + +### Work items + +1. **Add or extend an agent validation command.** + - Preferred: `baton agents doctor` or an enhanced `baton validate` report. + - Validate frontmatter shape, model value, permission mode, tool list, description length, and output-format section. + +2. **Verify knowledge references.** + - Check `knowledge_packs` frontmatter against loaded knowledge registry. + - Check “Before Starting” file paths exist when they are local paths. + +3. **Add safety warnings.** + - Flag implementer agents with broad tools and no clear need. + - Flag reviewers/auditors with `Write` or `Edit` unless explicitly justified. + - Flag very large baked-in knowledge sections. + +4. **Add machine-readable report.** + - Support `--json` output for CI and PMO future use. + +### Suggested files + +```text +agent_baton/cli/commands/agents/*.py +agent_baton/core/orchestration/registry.py +agent_baton/core/orchestration/knowledge_registry.py +tests/agents/test_agent_doctor.py +``` + +### Acceptance criteria + +- `baton agents doctor` or equivalent exits non-zero on broken required fields. +- Missing knowledge packs are reported with agent name and field. +- Safety warnings are visible but do not block unless `--strict` is passed. +- JSON output is stable enough for tests. + +### Validation commands + +```bash +python -m pytest -q tests/agents/test_agent_doctor.py +baton agents doctor --strict || true +baton agents doctor --json > /tmp/agent-doctor.json +``` + +### Baton run prompt + +```text +Implement Phase 2 of roadmaps/03-talent-builder-subagent-management.md. +Add a lightweight agent doctor that validates generated agents, knowledge references, and unsafe tool permissions. +Keep changes additive and backward compatible. +``` + +--- + +## Phase 3 — Add draft/review/promote workflow using metadata + +### Developer outcome + +New subagents can be created as drafts, reviewed, then promoted for use. Developers can distinguish experimental agents from approved team assets. + +### Work items + +1. **Support `status` metadata.** + - Recognize `status: draft|reviewed|approved|deprecated` in frontmatter. + - Default missing status to `approved` for backward compatibility, but warn for generated agents missing it. + +2. **Add doctor rules for lifecycle.** + - Draft agents should be visible but optionally excluded from planning unless explicitly requested. + - Deprecated agents should warn when selected. + +3. **Add promote checklist.** + - `baton agents promote ` may be a simple frontmatter edit, or document a manual process if command scope is too much. + - Require doctor pass before promotion. + +4. **Record generation provenance.** + - Encourage `created_by: talent-builder`, `source_docs`, and `version` fields. + +### Suggested files + +```text +agent_baton/core/orchestration/registry.py +agent_baton/cli/commands/agents/*.py +agents/talent-builder.md +templates/agents/*.md +tests/agents/ +``` + +### Acceptance criteria + +- Draft/deprecated status is visible in agent listing or doctor output. +- Planner behavior remains backward compatible for existing agents. +- A developer can promote a generated agent with a documented checklist. + +### Validation commands + +```bash +python -m pytest -q tests/agents +baton agents doctor --strict +``` + +### Baton run prompt + +```text +Implement Phase 3 of roadmaps/03-talent-builder-subagent-management.md. +Add lifecycle metadata support and draft/review/promote validation for generated agents. +Do not change the fundamental filesystem-backed registry model. +``` + +--- + +## Phase 4 — Make the agent catalog useful + +### Developer outcome + +Developers can browse available agents, see which are safe/approved, understand when to use them, and know what knowledge packs they depend on. + +### Work items + +1. **Improve agent listing output.** + - Show name, category, model, permission mode, status, knowledge packs, and source path. + +2. **Add catalog documentation.** + - Generate or update `docs/agent-roster.md` from registry data where practical. + +3. **Expose agent health in PMO/API.** + - Add status/knowledge metadata to the `/agents` response if low risk. + +4. **Add examples.** + - Include “create a new agent for X” and “validate generated agent” examples. + +### Suggested files + +```text +agent_baton/api/routes/agents.py +agent_baton/api/models/responses.py +agent_baton/cli/commands/agents/*.py +docs/agent-roster.md +pmo-ui/src/** +tests/api/ +tests/agents/ +``` + +### Acceptance criteria + +- CLI agent list is useful for a developer deciding which agent to use. +- Agent catalog docs include generated-agent workflow. +- API response remains backward compatible or versioned. + +### Validation commands + +```bash +python -m pytest -q tests/api tests/agents +baton agents list || true +``` + +### Baton run prompt + +```text +Implement Phase 4 of roadmaps/03-talent-builder-subagent-management.md. +Make the agent catalog more useful in CLI/API/docs with status, dependencies, and examples. +Keep schema changes backward compatible. +``` diff --git a/reference_docs/framing_and_roadmap/04-knowledge-pack-management.md b/reference_docs/framing_and_roadmap/04-knowledge-pack-management.md new file mode 100644 index 00000000..6a48a6d6 --- /dev/null +++ b/reference_docs/framing_and_roadmap/04-knowledge-pack-management.md @@ -0,0 +1,234 @@ +# Roadmap: Knowledge Pack Management + +**Capability goal:** Developers should be able to create, validate, attach, search, and maintain knowledge packs so agents have the right project/domain context without bloated prompts or missing references. + +**No-structural-refactor constraint:** Keep `KnowledgeRegistry`, `KnowledgeResolver`, `KnowledgePack`, and filesystem-backed `.claude/knowledge/` layout. Add default wiring, commands, validation, docs, and focused improvements only. + +--- + +## Phase 1 — Make knowledge packs active by default + +### Developer outcome + +When a project has knowledge packs, Baton uses them automatically and reports what was loaded. Developers do not need to know hidden constructor details. + +### Work items + +1. **Load `KnowledgeRegistry` in default planner paths.** + - CLI plan creation should load global and project knowledge packs. + - API/PMO planner construction should do the same. + +2. **Normalize manifest naming.** + - Pick `knowledge.yaml` as canonical. + - Update docs/model comments that mention `pack.yaml`. + - Optionally tolerate `pack.yaml` with a warning and migration hint. + +3. **Report loaded packs.** + - Plan diagnostics should include `knowledge_packs_loaded`, `degraded_packs`, `docs_indexed`, and `attachments_selected`. + +4. **Add a tiny sample pack.** + - Create a small example under docs or templates that users can copy. + +### Suggested files + +```text +agent_baton/core/orchestration/knowledge_registry.py +agent_baton/core/engine/planning/stages/risk.py +agent_baton/api/deps.py +agent_baton/models/knowledge.py +docs/orchestrator-usage.md +templates/knowledge/example-pack/ +tests/knowledge/ +``` + +### Acceptance criteria + +- A project-level pack in `.claude/knowledge//knowledge.yaml` is loaded by default. +- Manifest naming is consistent in code comments and docs. +- Missing manifests are reported as degraded but do not break planning. +- Tests cover global/project override and degraded pack reporting. + +### Validation commands + +```bash +python -m pytest -q tests/knowledge tests/planning +baton plan "Use the sample domain rules to update validation" --explain +``` + +### Baton run prompt + +```text +Implement Phase 1 of roadmaps/04-knowledge-pack-management.md. +Wire KnowledgeRegistry into default planning paths, normalize manifest naming, and report loaded/degraded knowledge packs. +Do not redesign the knowledge model. +``` + +--- + +## Phase 2 — Add knowledge doctor and search + +### Developer outcome + +Developers can validate knowledge packs before running agents and can search what Baton would know about a task. + +### Work items + +1. **Add `baton knowledge doctor`.** + - Validate manifests, document frontmatter, token estimates, duplicate doc names, missing files, empty descriptions, and oversized inline candidates. + - Support `--strict` and `--json`. + +2. **Add `baton knowledge search `.** + - Search the registry using current metadata TF-IDF. + - Show pack, doc, score, path, tags, priority, and token estimate. + +3. **Add attach simulation.** + - `baton knowledge resolve --agent --task ` should show which docs would attach and whether inline/reference. + - If full command is too much, add this as a doctor sub-mode. + +4. **Improve validation messages.** + - Every warning should tell the developer what to edit. + +### Suggested files + +```text +agent_baton/cli/commands/knowledge/*.py +agent_baton/core/orchestration/knowledge_registry.py +agent_baton/core/engine/knowledge_resolver.py +tests/knowledge/test_knowledge_doctor.py +tests/knowledge/test_knowledge_search.py +``` + +### Acceptance criteria + +- Doctor catches missing `knowledge.yaml`, empty doc metadata, and large inline candidates. +- Search returns useful metadata and paths. +- Resolve simulation matches actual resolver output for a fixture pack. + +### Validation commands + +```bash +python -m pytest -q tests/knowledge +baton knowledge doctor --strict || true +baton knowledge search "authentication token renewal" +``` + +### Baton run prompt + +```text +Implement Phase 2 of roadmaps/04-knowledge-pack-management.md. +Add knowledge doctor/search/resolve-simulation commands using existing KnowledgeRegistry and KnowledgeResolver. +Keep implementation additive and testable. +``` + +--- + +## Phase 3 — Turn knowledge gaps into improvement suggestions + +### Developer outcome + +When agents report missing knowledge, developers get concrete suggestions for what pack/doc to create or update. + +### Work items + +1. **Summarize knowledge gaps.** + - Add CLI/API output listing recent `KnowledgeGapRecord` items by agent, task type, and frequency. + +2. **Suggest pack updates.** + - For recurring gaps, suggest target pack name, doc name, tags, and draft grounding. + +3. **Link gaps to Talent Builder.** + - Add a suggested prompt: “Use talent-builder to create/update knowledge pack X with these gaps.” + +4. **Track knowledge usage/freshness where already supported.** + - Surface last-used and usage count if lifecycle telemetry exists. + - Do not make telemetry required for planning. + +### Suggested files + +```text +agent_baton/models/knowledge.py +agent_baton/core/engine/knowledge_gap.py +agent_baton/core/learn/** +agent_baton/cli/commands/knowledge/*.py +agents/talent-builder.md +tests/knowledge/ +``` + +### Acceptance criteria + +- Developers can list recent knowledge gaps. +- Recurring gaps produce actionable pack/doc suggestions. +- Talent Builder instructions include the gap-to-pack workflow. +- Planning still succeeds when telemetry tables are unavailable. + +### Validation commands + +```bash +python -m pytest -q tests/knowledge tests/learn +baton knowledge gaps || true +``` + +### Baton run prompt + +```text +Implement Phase 3 of roadmaps/04-knowledge-pack-management.md. +Surface knowledge gaps as actionable pack-update suggestions and connect the flow to Talent Builder. +Do not require new storage migrations unless absolutely necessary. +``` + +--- + +## Phase 4 — Make knowledge usable in UI and docs + +### Developer outcome + +Developers can understand and maintain project knowledge without reading implementation code or inspecting raw files manually. + +### Work items + +1. **Add knowledge docs.** + - Document pack structure, frontmatter, manifest fields, tags, priorities, grounding, delivery behavior, and token budgeting. + +2. **Add PMO/API read-only knowledge metadata.** + - Return pack/doc metadata for UI display. + - Avoid exposing full document content unless explicitly requested and safe. + +3. **Add PMO knowledge panel.** + - Small read-only list of packs, docs, degraded status, and token estimates. + +4. **Add examples.** + - Include at least one example pack and one task showing automatic attachment. + +### Suggested files + +```text +agent_baton/api/routes/** +agent_baton/api/models/responses.py +pmo-ui/src/** +docs/knowledge-packs.md +docs/orchestrator-usage.md +templates/knowledge/example-pack/ +tests/api/ +pmo-ui/src/**/*.test.tsx +``` + +### Acceptance criteria + +- Docs explain how knowledge gets from pack to agent prompt. +- PMO/API can list knowledge metadata. +- Example pack works with `baton plan --explain`. + +### Validation commands + +```bash +python -m pytest -q tests/api tests/knowledge +cd pmo-ui && npm run build && npm run test:run +``` + +### Baton run prompt + +```text +Implement Phase 4 of roadmaps/04-knowledge-pack-management.md. +Expose read-only knowledge metadata in docs/API/PMO and add example packs. +Do not expose arbitrary file contents through the API. +``` diff --git a/reference_docs/framing_and_roadmap/05-general-developer-ux-and-layout.md b/reference_docs/framing_and_roadmap/05-general-developer-ux-and-layout.md new file mode 100644 index 00000000..2bf5ba4f --- /dev/null +++ b/reference_docs/framing_and_roadmap/05-general-developer-ux-and-layout.md @@ -0,0 +1,238 @@ +# Roadmap: General Code Layout, Developer UX, and Release Polish + +**Capability goal:** Developers should be able to install, inspect, validate, and operate Baton with fewer surprises. This roadmap deliberately avoids structural refactoring and focuses on quick wins around diagnostics, terminology, packaging, CI smoke coverage, and documentation. + +**No-structural-refactor constraint:** Do not split large modules or reorganize package directories in this roadmap. Improve around the edges: commands, docs, tests, packaging manifests, and diagnostics. + +--- + +## Phase 1 — Add a developer-facing doctor and terminology cleanup + +### Developer outcome + +A developer can run one command and learn whether Baton is installed correctly, whether agents/knowledge/packs are discoverable, whether the PMO UI assets exist, and which optional features are degraded. + +### Work items + +1. **Add `baton doctor`.** + - Check Python version, package version, bundled agents, project agents, knowledge packs, assurance packs, PMO UI assets, `bd` availability, git repo status, Claude CLI availability, and writable `.claude/team-context`. + - Support `--json`. + +2. **Clean up terminology.** + - Standardize `talent-builder` vs `talent-manager`. + - Standardize `knowledge.yaml` naming. + - Distinguish knowledge packs vs assurance packs in docs. + +3. **Audit package resources.** + - Report whether bundled agents, references, templates, and PMO static assets are available in a wheel install. + - Do not change packaging yet unless the fix is trivial. + +4. **Add Makefile targets if missing.** + - Add `lint`, `typecheck`, `doctor`, and `ci-local` as wrappers if tooling is available. + +### Suggested files + +```text +agent_baton/cli/commands/**/doctor*.py +agent_baton/cli/main.py +pyproject.toml +Makefile +docs/*.md +README.md +tests/cli/test_doctor.py +``` + +### Acceptance criteria + +- `baton doctor` produces human-readable and JSON reports. +- Terminology inconsistencies are cleaned or explicitly aliased. +- Doctor reports missing optional features as warnings, not crashes. + +### Validation commands + +```bash +python -m pytest -q tests/cli/test_doctor.py +baton doctor +baton doctor --json > /tmp/baton-doctor.json +``` + +### Baton run prompt + +```text +Implement Phase 1 of roadmaps/05-general-developer-ux-and-layout.md. +Add a developer-facing doctor command, terminology cleanup, and package-resource checks. +Do not reorganize directories or split large modules. +``` + +--- + +## Phase 2 — Add quick CI and smoke coverage for developer outcomes + +### Developer outcome + +Developers can trust that core workflows still work after changes: help output, plan creation, agent validation, knowledge loading, PMO UI build, and package build. + +### Work items + +1. **Add CLI smoke tests.** + - `baton --help` + - `baton doctor --json` + - `baton validate agents` + - `baton plan` smoke with deterministic fallback path. + +2. **Add package build smoke.** + - Build wheel, install into clean venv, run `baton --help` and `baton doctor`. + +3. **Add UI build smoke.** + - Ensure PMO UI build/test runs in a separate job or release check. + +4. **Add planner golden smoke target.** + - Not exhaustive; just enough to catch import/runtime asset breakage. + +### Suggested files + +```text +.github/workflows/tests.yml +.github/workflows/release-pypi.yml +Makefile +tests/cli/ +tests/packaging/ +pmo-ui/package.json +``` + +### Acceptance criteria + +- CI catches package import failures and missing runtime resources. +- UI build is exercised at least on PR or release branches. +- Wheel install smoke proves source checkout is not required for basic use. + +### Validation commands + +```bash +python -m pytest -q tests/cli tests/packaging +python -m build +cd pmo-ui && npm run build && npm run test:run +``` + +### Baton run prompt + +```text +Implement Phase 2 of roadmaps/05-general-developer-ux-and-layout.md. +Add CI and smoke tests that protect end-user workflows, including wheel install and PMO UI build. +Do not broaden into full module refactoring. +``` + +--- + +## Phase 3 — Improve PMO/client operational polish + +### Developer outcome + +Developers using the PMO UI get clearer errors and a working path when API auth or capability diagnostics are enabled. + +### Work items + +1. **Centralize PMO API client behavior.** + - Ensure auth/header injection is consistent. + - Ensure request timeout and error handling are consistent. + - Avoid raw `fetch()` calls where the shared request wrapper should be used. + +2. **Show capability health in PMO.** + - Display doctor summary or a subset: agents loaded, knowledge packs loaded, planner hard-gate mode, team backend, PMO API version. + +3. **Improve API error display.** + - Show stable error messages in UI instead of raw exception blobs. + +4. **Add tests for API client behavior.** + - Mock 401/403/500 and timeout. + +### Suggested files + +```text +pmo-ui/src/api/client.ts +pmo-ui/src/api/types.ts +pmo-ui/src/** +agent_baton/api/routes/** +agent_baton/api/models/responses.py +pmo-ui/src/**/*.test.tsx +tests/api/ +``` + +### Acceptance criteria + +- PMO client uses a central request/auth/error path. +- Capability health is visible in UI or an API response consumed by UI. +- UI tests cover auth and error states. + +### Validation commands + +```bash +python -m pytest -q tests/api +cd pmo-ui && npm run build && npm run test:run +``` + +### Baton run prompt + +```text +Implement Phase 3 of roadmaps/05-general-developer-ux-and-layout.md. +Improve PMO client consistency and capability health visibility. +Do not redesign the UI or API route layout. +``` + +--- + +## Phase 4 — Release and documentation hardening + +### Developer outcome + +Developers can install Baton, follow docs, validate their project, and run the improved capabilities without guessing which assets or commands are available. + +### Work items + +1. **Add release checklist.** + - Build wheel, install wheel, run doctor, run plan smoke, run agent doctor, run knowledge doctor, run PMO UI build. + +2. **Update CLI reference.** + - Include `doctor`, `agents doctor`, `knowledge doctor/search`, team backend diagnostics, and plan diagnostics. + +3. **Add “first 15 minutes” developer guide.** + - Install, run doctor, validate agents, create a plan, inspect diagnostics, execute dry-run, add knowledge pack. + +4. **Add examples.** + - Example plan with knowledge pack. + - Example generated agent lifecycle. + - Example team execution report. + +### Suggested files + +```text +docs/cli-reference.md +docs/getting-started.md +docs/knowledge-packs.md +docs/agent-roster.md +docs/engine-and-runtime.md +.github/workflows/release-pypi.yml +README.md +``` + +### Acceptance criteria + +- Docs cover every new command added by these roadmaps. +- Release workflow or checklist includes package and capability smoke tests. +- A new developer can follow the getting-started guide without using source-only assumptions. + +### Validation commands + +```bash +mkdocs build --strict || true +python -m build +baton doctor +``` + +### Baton run prompt + +```text +Implement Phase 4 of roadmaps/05-general-developer-ux-and-layout.md. +Harden docs and release checks for the newly improved developer-facing capabilities. +Do not perform structural refactoring. +``` diff --git a/references/CLAUDE.md b/references/CLAUDE.md index 12ac0fa9..b609d240 100644 --- a/references/CLAUDE.md +++ b/references/CLAUDE.md @@ -1,6 +1,6 @@ # references/ — distributable reference procedures -18 reference procedures installed into user projects under `.claude/references/`. Cross-cutting rules: [../CLAUDE.md](../CLAUDE.md). +20 reference procedures installed into user projects under `.claude/references/`. Cross-cutting rules: [../CLAUDE.md](../CLAUDE.md). ## What a reference is @@ -16,12 +16,14 @@ frameworks, escalation chains, formatting standards. | Engine protocol from the agent side | `baton-engine.md` | | Common Baton patterns | `baton-patterns.md` | | Adaptive execution heuristics | `adaptive-execution.md` | +| Agent authoring contract for generated agents | `agent-authoring.md` | | Agent routing rules | `agent-routing.md` | | Communication protocols | `comms-protocols.md` | | Compliance & audit chain | `compliance-audit-chain.md` | | Cost / budget rules | `cost-budget.md` | | Decision framework | `decision-framework.md` | | Doc generation conventions | `doc-generation.md` | +| Evidence bundles (produce, verify, interpret) | `evidence-bundle.md` | | Failure handling | `failure-handling.md` | | Git strategy | `git-strategy.md` | | Guardrail presets | `guardrail-presets.md` | diff --git a/references/agent-authoring.md b/references/agent-authoring.md new file mode 100644 index 00000000..6f93cc6a --- /dev/null +++ b/references/agent-authoring.md @@ -0,0 +1,77 @@ +--- +name: agent-authoring +description: Standard contract for authoring generated Agent Baton agent files. +--- + +# Agent Authoring + +Use this reference when `talent-builder` creates or updates an agent. + +## Generated-Agent Contract + +Every generated agent is a markdown file with YAML frontmatter and a body +prompt. The frontmatter is the routing surface. The body is the operating +contract the agent follows at dispatch time. + +Required frontmatter fields: +- `name` +- `description` +- `model` +- `permissionMode` +- `tools` + +Recommended frontmatter fields: +- `owner` +- `status` +- `version` +- `created_by` +- `last_reviewed` +- `knowledge_packs` + +Required body sections: +- Mission +- Before Starting +- Knowledge References +- Principles +- Anti-Patterns +- Output Format + +## Field Guidance + +| Field | Guidance | +|-------|----------| +| `name` | Kebab-case role name. Use `role--flavor` for variants. | +| `description` | Multi-line trigger guidance. Say when to use the agent and when not to. | +| `model` | Use `opus` for high-judgment reasoning, `sonnet` for implementation, `haiku` for narrow procedural tasks. | +| `permissionMode` | Use `default` for reviewers/advisors and `auto-edit` only for trusted implementers. | +| `tools` | Start with the minimum viable set. Reviewers usually need only `Read`, `Glob`, `Grep`. | +| `owner` | Team or person responsible for maintenance. | +| `status` | One of `draft`, `active`, `deprecated`, or `archived`. | +| `version` | Semantic version for prompt contract changes. | +| `created_by` | Usually `talent-builder`, unless another agent or human authored it. | +| `last_reviewed` | ISO date for the last contract review. | +| `knowledge_packs` | List of knowledge-pack paths the agent is expected to read. Use `[]` when none are required. | + +## Tool Policy + +Avoid broad tools unless the mission requires them. Add `Edit`, `Write`, +`Bash`, or external MCP/server tools only when the agent's responsibilities +cannot be completed with read-only tools. When broad tools are included, state +the reason in the Principles or Before Starting section. + +## Reference Validation + +Before saving or reporting an agent: +- Read back the final agent file. +- Validate references named in `knowledge_packs` and Knowledge References. +- Remove stale paths, or mark optional references explicitly with why they are + optional. +- Keep generated-agent prompts concise enough that dispatch context is not + dominated by static boilerplate. + +## Starter Templates + +Use these files as copy sources: +- `.claude/templates/agents/base-agent.md` +- `.claude/templates/agents/flavored-agent.md` +- `.claude/templates/agents/reviewer-agent.md` diff --git a/scripts/install.ps1 b/scripts/install.ps1 index e06545ae..8ecd3f3f 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -23,6 +23,7 @@ $RootDir = Split-Path -Parent $ScriptDir $AgentsDir = Join-Path $RootDir "agents" $RefsDir = Join-Path $RootDir "references" $SkillsSrc = Join-Path $RootDir "templates" "skills" +$AgentTemplatesSrc = Join-Path $RootDir "templates" "agents" $ClaudeMd = Join-Path $RootDir "templates" "CLAUDE.md" $SettingsJ = Join-Path $RootDir "templates" "settings.json" @@ -86,6 +87,7 @@ $RefTarget = Join-Path $Base "references" $TeamCtx = Join-Path $Base "team-context" $KnowledgeDir = Join-Path $Base "knowledge" $SkillsDir = Join-Path $Base "skills" +$TemplateAgentTarget = Join-Path $Base "templates\agents" # Test write permissions try { @@ -123,6 +125,16 @@ Get-ChildItem "$RefsDir\*.md" | ForEach-Object { New-Item -ItemType Directory -Force -Path $TeamCtx | Out-Null New-Item -ItemType Directory -Force -Path $KnowledgeDir | Out-Null New-Item -ItemType Directory -Force -Path $SkillsDir | Out-Null +New-Item -ItemType Directory -Force -Path $TemplateAgentTarget | Out-Null + +$templateAgentCount = 0 +if (Test-Path $AgentTemplatesSrc) { + Get-ChildItem "$AgentTemplatesSrc\*.md" | ForEach-Object { + Copy-Item $_.FullName -Destination $TemplateAgentTarget -Force + Write-Host " + Agent template: $($_.Name)" -ForegroundColor Green + $templateAgentCount++ + } +} # Install skills from templates/skills/ $skillCount = 0 @@ -136,7 +148,7 @@ if (Test-Path $SkillsSrc) { } } -Write-Host " + Dirs: team-context/, knowledge/, skills/" -ForegroundColor Green +Write-Host " + Dirs: team-context/, knowledge/, skills/, templates/agents/" -ForegroundColor Green # CLAUDE.md — skip on upgrade, but merge identity block if missing if ($Upgrade) { @@ -265,7 +277,7 @@ open(sys.argv[2], 'w').write(json.dumps(dst, indent=2) + '\n') } Write-Host "" -Write-Host " Installed: $agentCount agents + $refCount references + $skillCount skills" -ForegroundColor Green +Write-Host " Installed: $agentCount agents + $refCount references + $skillCount skills + $templateAgentCount agent templates" -ForegroundColor Green # ── Step 3: MCP / Knowledge Infrastructure ───────────────── Write-Host "" diff --git a/scripts/install.sh b/scripts/install.sh index 09838ac8..c32697ba 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -7,6 +7,7 @@ ROOT_DIR="$(dirname "$SCRIPT_DIR")" AGENTS_DIR="$ROOT_DIR/agents" REFS_DIR="$ROOT_DIR/references" SKILLS_SRC="$ROOT_DIR/templates/skills" +AGENT_TEMPLATES_SRC="$ROOT_DIR/templates/agents" CLAUDE_MD="$ROOT_DIR/templates/CLAUDE.md" SETTINGS_JSON="$ROOT_DIR/templates/settings.json" @@ -88,6 +89,7 @@ REF_TARGET="$BASE/references" TEAM_CTX="$BASE/team-context" KNOWLEDGE_DIR="$BASE/knowledge" SKILLS_DIR="$BASE/skills" +TEMPLATE_AGENT_TARGET="$BASE/templates/agents" # Pre-flight: verify write permissions if ! mkdir -p "$BASE" 2>/dev/null; then @@ -109,7 +111,7 @@ echo "" echo " STEP 2: Installing Core Files" echo " ─────────────────────────────" -mkdir -p "$AGENT_TARGET" "$REF_TARGET" "$TEAM_CTX" "$KNOWLEDGE_DIR" "$SKILLS_DIR" +mkdir -p "$AGENT_TARGET" "$REF_TARGET" "$TEAM_CTX" "$KNOWLEDGE_DIR" "$SKILLS_DIR" "$TEMPLATE_AGENT_TARGET" agent_count=0 for f in "$AGENTS_DIR"/*.md; do @@ -125,6 +127,16 @@ for f in "$REFS_DIR"/*.md; do ref_count=$((ref_count + 1)) done +template_agent_count=0 +if [ -d "$AGENT_TEMPLATES_SRC" ]; then + for f in "$AGENT_TEMPLATES_SRC"/*.md; do + [ -f "$f" ] || continue + cp "$f" "$TEMPLATE_AGENT_TARGET/" + echo " + Agent template: $(basename "$f")" + template_agent_count=$((template_agent_count + 1)) + done +fi + # Install skills from templates/skills/ skill_count=0 if [ -d "$SKILLS_SRC" ]; then @@ -139,7 +151,7 @@ if [ -d "$SKILLS_SRC" ]; then done fi -echo " + Dirs: team-context/, knowledge/, skills/" +echo " + Dirs: team-context/, knowledge/, skills/, templates/agents/" # CLAUDE.md — skip on upgrade, but merge identity block if missing if [ "$UPGRADE" = true ]; then @@ -248,7 +260,7 @@ print(' merge: settings.json hooks (' + str(len(src_hooks)) + ' events)') fi echo "" -echo " Installed: $agent_count agents + $ref_count references + $skill_count skills" +echo " Installed: $agent_count agents + $ref_count references + $skill_count skills + $template_agent_count agent templates" # ── Step 3: Knowledge Infrastructure ────────────────────── echo "" diff --git a/templates/CLAUDE.md b/templates/CLAUDE.md index c82c4a8e..2768b965 100644 --- a/templates/CLAUDE.md +++ b/templates/CLAUDE.md @@ -257,7 +257,9 @@ specialist agents. For simple, single-domain tasks (bug fixes, small features, utility functions), work directly without the orchestrator. -## Assurance Surface +## Regulated Domain Guardrails + +For regulated-domain work only — these guardrails (the README's Pillar 4) keep high-risk, audit-controlled changes from diverging. Org-level assurance packs are stored in `.claude/packs/`. Validate the active pack configuration with: diff --git a/templates/agents/base-agent.md b/templates/agents/base-agent.md new file mode 100644 index 00000000..3af574da --- /dev/null +++ b/templates/agents/base-agent.md @@ -0,0 +1,56 @@ +--- +name: base-agent +description: | + Starter for an unflavored specialist agent. Replace this text with specific + trigger conditions, including when the agent should not be used. +model: sonnet +permissionMode: auto-edit +tools: Read, Glob, Grep +owner: unassigned +status: draft +version: 0.1.0 +created_by: talent-builder +last_reviewed: 2026-07-02 +knowledge_packs: [] +--- + +# Base Agent + +## Mission + +You are a focused specialist. Replace this paragraph with the agent's exact +work product, decision boundary, and success criteria. + +## Before Starting + +1. Read this entire agent definition. +2. Read back every file listed under Knowledge References. +3. Validate references exist before relying on them; report missing references + instead of inventing context. + +## Knowledge References + +- Add required `.claude/knowledge/...` or `references/...` paths here. +- If no external references are required, keep `knowledge_packs: []` and state + that the agent is prompt-only. + +## Principles + +- Stay inside the role boundary. +- Prefer project conventions over generic patterns. +- Use the least privileged tool set that can complete the mission. + +## Anti-Patterns + +- Do not broaden scope into orchestration, planning, or unrelated refactors. +- Do not use broad tools such as `Edit`, `Write`, or `Bash` unless the agent + contract explicitly adds them. +- Do not cite references that were not read back and validated. + +## Output Format + +Return: +1. Work completed or recommendation. +2. Files or references used. +3. Decisions and rationale. +4. Open questions or blockers. diff --git a/templates/agents/flavored-agent.md b/templates/agents/flavored-agent.md new file mode 100644 index 00000000..058d96da --- /dev/null +++ b/templates/agents/flavored-agent.md @@ -0,0 +1,57 @@ +--- +name: base-agent--flavor +description: | + Starter for a flavored variant of a base agent. Use instead of the base + agent when the task is clearly in this stack, domain, or workflow flavor. +model: sonnet +permissionMode: auto-edit +tools: Read, Glob, Grep +owner: unassigned +status: draft +version: 0.1.0 +created_by: talent-builder +last_reviewed: 2026-07-02 +knowledge_packs: [] +--- + +# Base Agent Flavor + +## Mission + +You are the flavored variant of `base-agent`. Replace this paragraph with the +specific framework, stack, domain, or workflow expertise that changes how the +base role operates. + +## Before Starting + +1. Read the base agent contract if it exists. +2. Read back every file listed under Knowledge References. +3. Validate references exist before relying on them; report missing references + instead of inventing context. + +## Knowledge References + +- Add flavor-specific `.claude/knowledge/...` or `references/...` paths here. +- Keep shared role guidance in the base agent unless this flavor intentionally + overrides it. + +## Principles + +- Keep the base role's output format unless the caller asks for a different one. +- Explain flavor-specific tradeoffs in the language of the base role. +- Prefer narrowly scoped tools; add broad tools only for explicit workflow need. + +## Anti-Patterns + +- Do not duplicate large base-agent guidance that can be referenced instead. +- Do not route generic base-role work to this flavor without a concrete flavor + signal. +- Do not cite references that were not read back and validated. + +## Output Format + +Return: +1. Flavor-specific assessment or work completed. +2. Files or references used. +3. Decisions and rationale. +4. Base-agent behavior overridden, if any. diff --git a/templates/agents/reviewer-agent.md b/templates/agents/reviewer-agent.md new file mode 100644 index 00000000..593e5c1c --- /dev/null +++ b/templates/agents/reviewer-agent.md @@ -0,0 +1,55 @@ +--- +name: reviewer-agent +description: | + Starter for a read-only reviewer or auditor agent. Use when the agent should + assess work, identify risks, and recommend changes without mutating files. +model: opus +permissionMode: default +tools: Read, Glob, Grep +owner: unassigned +status: draft +version: 0.1.0 +created_by: talent-builder +last_reviewed: 2026-07-02 +knowledge_packs: [] +--- + +# Reviewer Agent + +## Mission + +You are an independent reviewer. Replace this paragraph with the exact quality, +safety, compliance, or domain lens this agent applies. + +## Before Starting + +1. Read this entire agent definition. +2. Read back every file listed under Knowledge References. +3. Validate references exist before relying on them; report missing references + instead of inventing context. + +## Knowledge References + +- Add review rubric, policy, or domain reference paths here. +- Keep `tools` read-only unless the reviewer is explicitly allowed to patch. + +## Principles + +- Lead with findings ordered by severity. +- Ground every finding in a file, reference, or observable behavior. +- Separate confirmed defects from open questions. + +## Anti-Patterns + +- Do not rewrite implementation as part of review unless the contract changes + this agent into an implementer. +- Do not flag stylistic preferences as defects. +- Do not cite references that were not read back and validated. + +## Output Format + +Return: +1. Findings by severity. +2. Evidence and affected paths. +3. Open questions or assumptions. +4. Residual risk if no findings are present. diff --git a/tests/agents/test_generated_agent_contract.py b/tests/agents/test_generated_agent_contract.py new file mode 100644 index 00000000..de502361 --- /dev/null +++ b/tests/agents/test_generated_agent_contract.py @@ -0,0 +1,120 @@ +"""Contract tests for generated agent authoring assets.""" +from __future__ import annotations + +from pathlib import Path + +from agent_baton.core.orchestration.registry import AgentRegistry +from agent_baton.utils.frontmatter import parse_frontmatter + + +ROOT = Path(__file__).resolve().parents[2] +REQUIRED_FIELDS = ("name", "description", "model", "permissionMode", "tools") +RECOMMENDED_FIELDS = ( + "owner", + "status", + "version", + "created_by", + "last_reviewed", + "knowledge_packs", +) +REQUIRED_SECTIONS = ( + "Mission", + "Before Starting", + "Knowledge References", + "Principles", + "Anti-Patterns", + "Output Format", +) +TEMPLATE_FILES = ("base-agent.md", "flavored-agent.md", "reviewer-agent.md") + + +def _read(relative_path: str) -> str: + return (ROOT / relative_path).read_text(encoding="utf-8") + + +def _assert_contract_fields(text: str) -> None: + for field in REQUIRED_FIELDS: + assert field in text, f"missing required field {field}" + for field in RECOMMENDED_FIELDS: + assert field in text, f"missing recommended field {field}" + + +def _assert_contract_sections(text: str) -> None: + for section in REQUIRED_SECTIONS: + assert f"## {section}" in text or f"- {section}" in text, ( + f"missing generated-agent section {section}" + ) + + +def test_talent_builder_instructions_define_generated_agent_contract() -> None: + text = _read("agents/talent-builder.md") + + assert "Generated-Agent Contract" in text + _assert_contract_fields(text) + _assert_contract_sections(text) + assert "Avoid broad tools" in text + assert "read back" in text.lower() + assert "validate references" in text.lower() + + +def test_bundled_talent_builder_matches_source_agent() -> None: + source = ROOT / "agents" / "talent-builder.md" + bundled = ROOT / "agent_baton" / "_bundled_agents" / "talent-builder.md" + + assert bundled.exists() + assert bundled.read_text(encoding="utf-8") == source.read_text(encoding="utf-8") + + +def test_agent_authoring_reference_defines_contract() -> None: + text = _read("references/agent-authoring.md") + + assert "# Agent Authoring" in text + # talent-manager was never resolvable in the AgentRegistry; docs must + # not advertise the alias. + assert "talent-manager" not in text + assert "`talent-builder`" in text + _assert_contract_fields(text) + _assert_contract_sections(text) + + +def test_agent_roster_links_contract_without_phantom_alias() -> None: + text = _read("docs/agent-roster.md") + + assert "references/agent-authoring.md" in text + assert "permissionMode" in text + assert "talent-manager" not in text + assert "`talent-builder`" in text + + +def test_starter_templates_exist_and_match_generated_agent_contract() -> None: + template_dir = ROOT / "templates" / "agents" + + for filename in TEMPLATE_FILES: + path = template_dir / filename + assert path.exists(), f"missing template {filename}" + content = path.read_text(encoding="utf-8") + metadata, body = parse_frontmatter(content) + + for field in REQUIRED_FIELDS: + assert metadata.get(field), f"{filename} missing required {field}" + for field in RECOMMENDED_FIELDS: + assert field in metadata, f"{filename} missing recommended {field}" + _assert_contract_sections(body) + + +def test_existing_bundled_agents_still_parse() -> None: + bundled_dir = ROOT / "agent_baton" / "_bundled_agents" + registry = AgentRegistry() + parsed_names: list[str] = [] + + for path in sorted(bundled_dir.glob("*.md")): + if path.name == "CLAUDE.md": + continue + agent = registry._parse_agent_content(path.read_text(encoding="utf-8"), path.name) + assert agent is not None, f"failed to parse {path.name}" + assert agent.name + assert agent.description + parsed_names.append(agent.name) + + assert "talent-builder" in parsed_names + assert len(parsed_names) == 30 diff --git a/tests/api/test_specs_api.py b/tests/api/test_specs_api.py index 97c1c21a..b357a005 100644 --- a/tests/api/test_specs_api.py +++ b/tests/api/test_specs_api.py @@ -30,6 +30,10 @@ from fastapi.testclient import TestClient # noqa: E402 from agent_baton.api.server import create_app # noqa: E402 +from agent_baton.core.engine.planning.stages.validation import ( # noqa: E402 + PlanDefect, + PlanQualityError, +) from agent_baton.core.federate.spec_draft_store import SpecDraftStore # noqa: E402 @@ -59,6 +63,18 @@ def client(tmp_path: Path, spec_db: Path, monkeypatch): return TestClient(app) +def _plan_quality_error() -> PlanQualityError: + defect = PlanDefect( + code="audit_missing", + severity="critical", + message=( + "Compliance plans require Audit coverage. " + "Remediation: add a terminal Audit phase with an auditor step." + ), + ) + return PlanQualityError("Plan blocked by ValidationStage", defects=[defect]) + + @pytest.fixture() def store(spec_db: Path) -> SpecDraftStore: """Direct store access for seeding test data.""" @@ -287,6 +303,36 @@ def test_fire_approved_spec(self, client, store): assert refreshed.status == "fired" assert refreshed.task_id == "fire-task-001" + def test_fire_plan_quality_error_returns_422(self, client, store): + from agent_baton.models.spec_draft import ReviewData + + draft = store.create(title="Fire gated spec", body="body") + store.update_enrichment(draft.id, _mock_enrichment()) + store.update_status( + draft.id, + "approved", + review=ReviewData(action="approved", actor="bob"), + ) + + mock_forge = MagicMock() + mock_forge.create_plan.side_effect = _plan_quality_error() + + mock_pmo_store = MagicMock() + mock_project = MagicMock() + mock_pmo_store.get_project.return_value = mock_project + + with patch("agent_baton.api.deps.get_forge_session", return_value=mock_forge), \ + patch("agent_baton.api.deps.get_pmo_store", return_value=mock_pmo_store): + r = client.post( + f"/api/v1/pmo/specs/{draft.id}/fire", + json={"project_id": "proj-1"}, + ) + + assert r.status_code == 422, r.text + detail = r.json()["detail"] + assert detail["error"] == "plan_quality_error" + assert detail["defects"][0]["code"] == "audit_missing" + # --------------------------------------------------------------------------- # Test 11: POST /pmo/specs/import (ADO unconfigured) → 501 diff --git a/tests/cli/test_doctor.py b/tests/cli/test_doctor.py new file mode 100644 index 00000000..77c9f27c --- /dev/null +++ b/tests/cli/test_doctor.py @@ -0,0 +1,998 @@ +"""Tests for the top-level ``baton doctor`` command.""" +from __future__ import annotations + +import argparse +import json +import sqlite3 +from pathlib import Path + +import pytest +import yaml + + +def _run_cli(argv: list[str]) -> int: + from agent_baton.cli.main import main + + try: + main(argv) + return 0 + except SystemExit as exc: + return int(exc.code) if exc.code is not None else 0 + + +def _write_agent(path: Path, *, name: str) -> None: + path.write_text( + ( + "---\n" + f"name: {name}\n" + "description: Test agent\n" + "model: sonnet\n" + "---\n" + "Test instructions.\n" + ), + encoding="utf-8", + ) + + +def _write_project_layout(root: Path) -> None: + agents_dir = root / ".claude" / "agents" + agents_dir.mkdir(parents=True) + _write_agent(agents_dir / "architect.md", name="architect") + + knowledge_pack = root / ".claude" / "knowledge" / "project-knowledge" + knowledge_pack.mkdir(parents=True) + (knowledge_pack / "knowledge.yaml").write_text( + yaml.safe_dump( + { + "name": "project-knowledge", + "description": "Project knowledge pack", + "tags": ["project"], + "default_delivery": "reference", + }, + sort_keys=False, + ), + encoding="utf-8", + ) + (knowledge_pack / "guide.md").write_text( + ( + "---\n" + "name: guide\n" + "description: Project guide\n" + "---\n" + "Body.\n" + ), + encoding="utf-8", + ) + + assurance_pack = root / ".claude" / "packs" / "project-assurance" + assurance_pack.mkdir(parents=True) + (assurance_pack / "pack.json").write_text( + '{"name": "project-assurance", "version": "0.1.0"}\n', + encoding="utf-8", + ) + + team_context = root / ".claude" / "team-context" + team_context.mkdir(parents=True) + + +def _write_beads_workspace(root: Path) -> Path: + beads_dir = root / ".beads" + beads_dir.mkdir(parents=True) + for name in ("config.yaml", "interactions.jsonl", "metadata.json"): + (beads_dir / name).write_text("{}\n", encoding="utf-8") + return beads_dir + + +def _valid_saved_plan(task_summary: str) -> dict[str, object]: + return { + "task_summary": task_summary, + "task_type": "documentation", + "complexity": "medium", + "risk_level": "LOW", + "phases": [ + { + "name": "Review", + "steps": [ + { + "task_description": "Inspect current versions", + "agent_name": "auditor", + "team": [], + } + ], + } + ], + } + + +def _check(payload: dict[str, object], check_id: str) -> dict[str, object]: + checks = payload["checks"] + assert isinstance(checks, list) + for check in checks: + assert isinstance(check, dict) + if check.get("id") == check_id: + return check + raise AssertionError(f"missing doctor check: {check_id}") + + +@pytest.fixture(autouse=True) +def _clear_baton_task_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BATON_TASK_ID", raising=False) + + +def test_discovery_registers_top_level_doctor_and_knowledge_doctor_separately( + capsys, +) -> None: + from agent_baton.cli.main import discover_commands + + modules = discover_commands() + + assert "diagnostics_cmd" in modules + assert modules["diagnostics_cmd"].__name__.endswith(".diagnostics_cmd") + assert "doctor_cmd" in modules + assert modules["doctor_cmd"].__name__.endswith(".knowledge.doctor_cmd") + + assert _run_cli(["doctor", "--help"]) == 0 + assert _run_cli(["knowledge", "doctor", "--help"]) == 0 + assert _run_cli(["--help"]) == 0 + help_text = capsys.readouterr().out + assert "doctor" in help_text + + +def test_doctor_json_reports_required_checks_and_optional_warnings( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + _write_project_layout(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("USERPROFILE", str(tmp_path / "home")) + monkeypatch.setenv("PATH", "") + + rc = _run_cli(["doctor", "--json"]) + payload = json.loads(capsys.readouterr().out) + check_ids = {check["id"] for check in payload["checks"]} + + assert rc == 0 + assert payload["schema_version"] == 1 + assert payload["ok"] is True + assert { + "python", + "package_version", + "bundled_agents", + "project_agents", + "knowledge_packs", + "assurance_packs", + "pmo_ui_assets", + "package_resources", + "bd", + "beads_workspace", + "git", + "git_worktree", + "claude_cli", + "team_context", + "planner_validation", + "terminology", + } <= check_ids + assert _check(payload, "bd")["status"] == "warning" + assert _check(payload, "claude_cli")["status"] == "warning" + assert _check(payload, "project_agents")["details"]["count"] == 1 + assert _check(payload, "knowledge_packs")["details"]["project_count"] == 1 + assert _check(payload, "assurance_packs")["details"]["project_count"] == 1 + assert _check(payload, "team_context")["status"] == "ok" + + bundled = _check(payload, "bundled_agents") + assert bundled["details"]["count"] > 0 + assert "talent-builder" in bundled["details"]["names"] + + resources = _check(payload, "package_resources") + assert { + "bundled_agents", + "references", + "templates", + "pmo_static_assets", + } <= set(resources["details"]["resources"]) + + +def test_doctor_handler_exits_nonzero_after_printing_error_payload( + monkeypatch: pytest.MonkeyPatch, + capsys, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + payload = { + "schema_version": 1, + "ok": False, + "project_root": "project", + "summary": {"ok": 1, "warning": 0, "error": 1}, + "checks": [ + { + "id": "planner_validation", + "label": "Planner validation", + "status": "error", + "message": "Saved plan validation failed", + "details": {}, + } + ], + } + monkeypatch.setattr( + diagnostics_cmd, + "build_report", + lambda project_root: payload, + ) + + with pytest.raises(SystemExit) as exc_info: + diagnostics_cmd.handler(argparse.Namespace(json=True)) + + assert exc_info.value.code == 1 + assert json.loads(capsys.readouterr().out) == payload + + +def test_beads_workspace_reports_ok_when_expected_files_exist( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + _write_beads_workspace(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "beads_workspace") + + assert check["status"] == "ok" + assert check["details"]["exists"] is True + assert check["details"]["missing_files"] == [] + assert sorted(check["details"]["present_files"]) == [ + "config.yaml", + "interactions.jsonl", + "metadata.json", + ] + + +def test_beads_workspace_reports_warning_when_workspace_is_missing( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "beads_workspace") + + assert check["status"] == "warning" + assert check["details"]["exists"] is False + assert check["details"]["missing_files"] == [ + "config.yaml", + "interactions.jsonl", + "metadata.json", + ] + assert check["details"]["present_files"] == [] + + +def test_git_worktree_reports_linked_worktree_metadata_when_git_is_monkeypatched( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + expected = { + ("rev-parse", "--is-inside-work-tree"): { + "returncode": 0, + "stdout": "true\n", + "stderr": "", + }, + ("rev-parse", "--show-superproject-working-tree"): { + "returncode": 0, + "stdout": "\n", + "stderr": "", + }, + ("rev-parse", "--git-dir"): { + "returncode": 0, + "stdout": ".git\\worktrees\\roadmap-ux-doctor\n", + "stderr": "", + }, + ("rev-parse", "--git-common-dir"): { + "returncode": 0, + "stdout": ".git\n", + "stderr": "", + }, + ("branch", "--show-current"): { + "returncode": 0, + "stdout": "bd-rm-ux-p1\n", + "stderr": "", + }, + ("rev-parse", "--abbrev-ref", "HEAD"): { + "returncode": 0, + "stdout": "bd-rm-ux-p1\n", + "stderr": "", + }, + } + + def fake_git(args: list[str], cwd: Path) -> dict[str, object]: + key = tuple(args) + if key not in expected: + raise AssertionError(f"unexpected git args: {args}") + return expected[key] + + monkeypatch.setattr(diagnostics_cmd.shutil, "which", lambda _name: "git") + monkeypatch.setattr(diagnostics_cmd, "_git", fake_git) + + check = diagnostics_cmd._check_git_worktree(tmp_path) + + assert check.status == "ok" + assert check.details["branch"] == "bd-rm-ux-p1" + assert check.details["git_dir"] == ".git\\worktrees\\roadmap-ux-doctor" + assert check.details["git_common_dir"] == ".git" + assert check.details["is_linked_worktree"] is True + assert check.details["is_submodule"] is False + assert check.details["detached_head"] is False + + +def test_git_helper_passes_no_optional_locks_before_subcommand( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + calls: list[list[str]] = [] + + class FakeCompletedProcess: + returncode = 0 + stdout = "" + stderr = "" + + def fake_run(cmd: list[str], **_kwargs: object) -> FakeCompletedProcess: + calls.append(cmd) + return FakeCompletedProcess() + + monkeypatch.setattr(diagnostics_cmd.subprocess, "run", fake_run) + + diagnostics_cmd._git(["status", "--porcelain"], tmp_path) + + assert calls == [ + [ + "git", + "-C", + str(tmp_path), + "--no-optional-locks", + "status", + "--porcelain", + ] + ] + + +def test_missing_optional_features_are_warnings_not_crashes( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + team_context.mkdir(parents=True) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + monkeypatch.setattr( + diagnostics_cmd, + "_probe_writable_directory", + lambda _path: (False, "permission denied"), + ) + + payload = diagnostics_cmd.build_report(tmp_path) + + assert payload["ok"] is True + assert _check(payload, "pmo_ui_assets")["status"] == "warning" + assert _check(payload, "bd")["status"] == "warning" + assert _check(payload, "claude_cli")["status"] == "warning" + assert _check(payload, "knowledge_packs")["status"] == "warning" + assert _check(payload, "assurance_packs")["status"] == "warning" + assert _check(payload, "team_context")["status"] == "warning" + assert _check(payload, "planner_validation")["status"] == "warning" + + +def test_knowledge_packs_warning_when_pack_dir_lacks_manifest( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + pack_dir = tmp_path / ".claude" / "knowledge" / "broken-pack" + pack_dir.mkdir(parents=True) + (pack_dir / "guide.md").write_text("body\n", encoding="utf-8") + (tmp_path / ".claude" / "team-context").mkdir(parents=True) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + monkeypatch.setattr( + diagnostics_cmd, + "_load_knowledge_registry_details", + lambda _root: { + "registry_loaded_count": 0, + "registry_well_formed_count": 0, + "registry_degraded_count": 1, + "registry_degraded_names": ["broken-pack"], + }, + ) + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "knowledge_packs") + + assert check["status"] == "warning" + assert check["details"]["project_count"] == 1 + assert check["details"]["project_with_manifest"] == 0 + assert check["details"]["registry_degraded_count"] == 1 + + +def test_assurance_packs_warning_when_validation_finds_invalid_pack( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + pack_dir = tmp_path / ".claude" / "packs" / "project-assurance" + pack_dir.mkdir(parents=True) + (pack_dir / "pack.json").write_text( + '{"name": "project-assurance", "version": "0.1.0"}\n', + encoding="utf-8", + ) + (tmp_path / ".claude" / "team-context").mkdir(parents=True) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + monkeypatch.setattr( + diagnostics_cmd, + "_validate_assurance_pack_dirs", + lambda *_roots: { + "invalid_count": 1, + "invalid_packs": [ + { + "pack": "project-assurance", + "path": str(pack_dir), + "errors": ["missing rubric"], + } + ], + }, + ) + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "assurance_packs") + + assert check["status"] == "warning" + assert check["details"]["project_count"] == 1 + assert check["details"]["invalid_count"] == 1 + + +def test_project_agents_warning_when_validation_raises( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + agents_dir = tmp_path / ".claude" / "agents" + agents_dir.mkdir(parents=True) + _write_agent(agents_dir / "architect.md", name="architect") + (tmp_path / ".claude" / "team-context").mkdir(parents=True) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + monkeypatch.setattr( + diagnostics_cmd, + "_validate_agent_dir", + lambda _path: { + "validated_count": 0, + "validation_warnings": 0, + "validation_errors": 0, + "validation_error": "validator import failed", + }, + ) + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "project_agents") + + assert check["status"] == "warning" + assert check["details"]["count"] == 1 + assert check["details"]["validation_error"] == "validator import failed" + + +def test_doctor_json_includes_planner_validation_warning_without_saved_plan( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + (tmp_path / ".claude" / "team-context").mkdir(parents=True) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + + assert check["status"] == "warning" + assert check["details"]["plan_path"] is None + assert check["details"]["machine_plan_importable"] is True + + +def test_doctor_build_report_does_not_create_team_context_artifacts_in_fresh_project( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + db_path = team_context / "baton.db" + wal_path = team_context / "baton.db-wal" + shm_path = team_context / "baton.db-shm" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + + assert check["status"] == "warning" + assert not (tmp_path / ".claude").exists() + assert not team_context.exists() + assert not db_path.exists() + assert not wal_path.exists() + assert not shm_path.exists() + + +def test_doctor_build_report_does_not_write_probe_files_in_team_context( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + team_context.mkdir(parents=True) + sentinel = team_context / "existing-note.txt" + sentinel.write_text("keep\n", encoding="utf-8") + before_children = sorted(path.name for path in team_context.iterdir()) + path_write_text = Path.write_text + + def guarded_write_text(self: Path, *args, **kwargs): + if self.parent == team_context and self.name.startswith(".baton-doctor"): + raise AssertionError("doctor attempted a temp-file writability probe") + return path_write_text(self, *args, **kwargs) + + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + monkeypatch.setattr(Path, "write_text", guarded_write_text) + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "team_context") + after_children = sorted(path.name for path in team_context.iterdir()) + + assert check["status"] == "ok" + assert before_children == after_children + assert not any(name.startswith(".baton-doctor") for name in after_children) + + +def test_doctor_discovers_task_scoped_saved_plan_for_planner_validation( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + task_dir = ( + tmp_path + / ".claude" + / "team-context" + / "executions" + / "task-002" + ) + task_dir.mkdir(parents=True) + (task_dir / "plan.json").write_text( + json.dumps(_valid_saved_plan("Review dependency versions")), + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + + assert check["message"] != "No saved plan is available to validate" + assert check["details"]["plan_path"] == str(task_dir / "plan.json") + assert check["details"]["plan_candidates"] == [ + str(tmp_path / ".claude" / "team-context" / "plan.json"), + str(tmp_path / "plan.json"), + str(task_dir / "plan.json"), + ] + + +def test_doctor_reads_active_task_from_existing_sqlite_without_extra_files( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + team_context.mkdir(parents=True) + task_a_dir = team_context / "executions" / "task-a" + task_b_dir = team_context / "executions" / "task-b" + task_a_dir.mkdir(parents=True) + task_b_dir.mkdir(parents=True) + (task_a_dir / "plan.json").write_text( + json.dumps(_valid_saved_plan("Task A plan")), + encoding="utf-8", + ) + (task_b_dir / "plan.json").write_text( + json.dumps(_valid_saved_plan("Task B plan")), + encoding="utf-8", + ) + db_path = team_context / "baton.db" + conn = sqlite3.connect(db_path) + assert conn.execute("PRAGMA journal_mode=WAL").fetchone()[0].lower() == "wal" + conn.execute( + "CREATE TABLE active_task (id INTEGER PRIMARY KEY, task_id TEXT)" + ) + conn.execute( + "INSERT INTO active_task (id, task_id) VALUES (1, 'task-b')" + ) + conn.commit() + conn.close() + assert not (team_context / "baton.db-wal").exists() + assert not (team_context / "baton.db-shm").exists() + before_paths = sorted( + str(path.relative_to(tmp_path)) + for path in tmp_path.rglob("*") + if path.is_file() + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + after_paths = sorted( + str(path.relative_to(tmp_path)) + for path in tmp_path.rglob("*") + if path.is_file() + ) + + assert check["status"] == "ok" + assert check["details"]["active_task_id"] == "task-b" + assert check["details"]["active_task_source"] == "sqlite" + assert check["details"]["plan_path"] == str(task_b_dir / "plan.json") + assert not (team_context / "active-task-id.txt").exists() + assert not (team_context / "baton.db-wal").exists() + assert not (team_context / "baton.db-shm").exists() + assert after_paths == before_paths + + +def test_doctor_reads_active_task_from_open_wal_sidecar_without_mutating_project( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + team_context.mkdir(parents=True) + task_dir = team_context / "executions" / "task-wal" + task_dir.mkdir(parents=True) + (task_dir / "plan.json").write_text( + json.dumps(_valid_saved_plan("Task WAL plan")), + encoding="utf-8", + ) + db_path = team_context / "baton.db" + conn = sqlite3.connect(db_path) + try: + assert conn.execute("PRAGMA journal_mode=WAL").fetchone()[0].lower() == "wal" + conn.execute( + "CREATE TABLE active_task (id INTEGER PRIMARY KEY, task_id TEXT)" + ) + conn.execute( + "INSERT INTO active_task (id, task_id) VALUES (1, 'task-wal')" + ) + conn.commit() + assert (team_context / "baton.db-wal").exists() + assert (team_context / "baton.db-shm").exists() + before_paths = sorted( + str(path.relative_to(tmp_path)) + for path in tmp_path.rglob("*") + if path.is_file() + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + after_paths = sorted( + str(path.relative_to(tmp_path)) + for path in tmp_path.rglob("*") + if path.is_file() + ) + finally: + conn.close() + + assert check["status"] == "ok" + assert check["details"]["active_task_id"] == "task-wal" + assert check["details"]["active_task_source"] == "sqlite" + assert check["details"]["plan_path"] == str(task_dir / "plan.json") + assert after_paths == before_paths + + +def test_doctor_records_degraded_sqlite_active_task_probe_details( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + team_context.mkdir(parents=True) + db_path = team_context / "baton.db" + db_path.write_text("not a sqlite database\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + probe = check["details"]["active_task_sqlite_probe"] + + assert check["status"] == "warning" + assert check["details"]["active_task_id"] is None + assert check["details"]["active_task_source"] is None + assert probe["status"] == "degraded" + assert probe["db_path"] == str(db_path) + assert probe["error"] + assert probe["error_type"] + + +def test_doctor_prefers_baton_task_id_env_over_sqlite_active_task( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + env_task_dir = team_context / "executions" / "task-env" + sqlite_task_dir = team_context / "executions" / "task-sqlite" + env_task_dir.mkdir(parents=True) + sqlite_task_dir.mkdir(parents=True) + (env_task_dir / "plan.json").write_text( + json.dumps(_valid_saved_plan("Task env plan")), + encoding="utf-8", + ) + (sqlite_task_dir / "plan.json").write_text( + json.dumps(_valid_saved_plan("Task sqlite plan")), + encoding="utf-8", + ) + db_path = team_context / "baton.db" + conn = sqlite3.connect(db_path) + conn.execute( + "CREATE TABLE active_task (id INTEGER PRIMARY KEY, task_id TEXT)" + ) + conn.execute( + "INSERT INTO active_task (id, task_id) VALUES (1, 'task-sqlite')" + ) + conn.commit() + conn.close() + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + monkeypatch.setenv("BATON_TASK_ID", " task-env ") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + + assert check["status"] == "ok" + assert check["details"]["active_task_id"] == "task-env" + assert check["details"]["active_task_source"] == "env" + assert check["details"]["plan_path"] == str(env_task_dir / "plan.json") + + +def test_doctor_prefers_active_task_scoped_plan_over_sorted_or_legacy_fallbacks( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + task_a_dir = team_context / "executions" / "task-a" + task_b_dir = team_context / "executions" / "task-b" + task_a_dir.mkdir(parents=True) + task_b_dir.mkdir(parents=True) + (team_context / "plan.json").write_text( + json.dumps(_valid_saved_plan("Legacy team-context plan")), + encoding="utf-8", + ) + (task_a_dir / "plan.json").write_text( + json.dumps(_valid_saved_plan("Task A plan")), + encoding="utf-8", + ) + (task_b_dir / "plan.json").write_text( + json.dumps(_valid_saved_plan("Task B plan")), + encoding="utf-8", + ) + (team_context / "active-task-id.txt").write_text("task-b\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + + assert check["status"] == "ok" + assert check["details"]["active_task_id"] == "task-b" + assert check["details"]["active_task_source"] == "file" + assert check["details"]["plan_path"] == str(task_b_dir / "plan.json") + assert check["details"]["plan_candidates"] == [ + str(team_context / "plan.json"), + str(tmp_path / "plan.json"), + str(task_a_dir / "plan.json"), + str(task_b_dir / "plan.json"), + ] + + +def test_doctor_reports_missing_active_task_plan_without_validating_fallback( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + fallback_plan = team_context / "plan.json" + fallback_plan.parent.mkdir(parents=True) + fallback_plan.write_text( + json.dumps(_valid_saved_plan("Legacy fallback plan")), + encoding="utf-8", + ) + missing_active_plan = ( + team_context / "executions" / "task-b" / "plan.json" + ) + (team_context / "active-task-id.txt").write_text("task-b\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + + assert check["status"] == "warning" + assert check["details"]["active_task_id"] == "task-b" + assert check["details"]["active_task_source"] == "file" + assert check["details"]["plan_path"] == str(missing_active_plan) + assert str(missing_active_plan) in check["message"] + assert check["details"]["plan_candidates"] == [ + str(team_context / "plan.json"), + str(tmp_path / "plan.json"), + ] + + +def test_doctor_reports_structured_error_for_malformed_saved_plan_json_shape( + tmp_path: Path, + monkeypatch, +) -> None: + from agent_baton.cli.commands import diagnostics_cmd + + home = tmp_path / "home" + home.mkdir() + team_context = tmp_path / ".claude" / "team-context" + team_context.mkdir(parents=True) + (team_context / "plan.json").write_text('{"phases": ["bad"]}\n', encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PATH", "") + + payload = diagnostics_cmd.build_report(tmp_path) + check = _check(payload, "planner_validation") + + assert check["status"] == "error" + assert check["details"]["plan_path"] == str(team_context / "plan.json") + assert check["details"]["validator_importable"] is True + assert "validation_error" in check["details"] + + +def test_doctor_text_distinguishes_pack_types_and_uses_canonical_terms( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + _write_project_layout(tmp_path) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("USERPROFILE", str(tmp_path / "home")) + + rc = _run_cli(["doctor"]) + out = capsys.readouterr().out + + assert rc == 0 + assert "Baton doctor" in out + assert "Knowledge packs" in out + assert "Assurance packs" in out + assert "talent-builder" in out + assert "knowledge.yaml" in out + assert "talent-manager" not in out + + +def test_doctor_help_mentions_json_and_pack_types(capsys) -> None: + rc = _run_cli(["doctor", "--help"]) + out = capsys.readouterr().out + + assert rc == 0 + assert "--json" in out + assert "knowledge packs" in out + assert "assurance packs" in out + + +def test_documented_terminology_is_canonical() -> None: + repo = Path(__file__).resolve().parents[2] + roster = (repo / "docs" / "agent-roster.md").read_text(encoding="utf-8") + terminology = (repo / "docs" / "terminology.md").read_text(encoding="utf-8") + governance = ( + repo / "docs" / "governance-knowledge-and-events.md" + ).read_text(encoding="utf-8") + cli_reference = (repo / "docs" / "cli-reference.md").read_text( + encoding="utf-8" + ) + + assert "`talent-builder`" in roster + # talent-manager was documented as an alias but never resolvable in the + # AgentRegistry; the claim was removed rather than implemented. + assert "talent-manager" not in roster + assert "knowledge.yaml" in terminology + assert "Knowledge pack" in terminology + assert "Assurance pack" in terminology + assert "knowledge.yaml" in governance + assert "assurance packs" in cli_reference diff --git a/tests/cli/test_plan_cmd_terse.py b/tests/cli/test_plan_cmd_terse.py index 912eff6c..a1474a7a 100644 --- a/tests/cli/test_plan_cmd_terse.py +++ b/tests/cli/test_plan_cmd_terse.py @@ -262,3 +262,57 @@ def test_no_save_emits_markdown(self, tmp_path: Path, capsys: pytest.CaptureFixt stdout = _run_handler(args, plan, ctx_dir, capsys) assert plan.to_markdown() in stdout + + def test_no_save_explain_emits_plan_explanation( + self, tmp_path: Path, capsys: pytest.CaptureFixture + ) -> None: + plan = _make_minimal_plan() + ctx_dir = tmp_path / ".claude" / "team-context" + ctx_dir.mkdir(parents=True) + args = _make_args(save=False, explain=True) + + stdout = _run_handler(args, plan, ctx_dir, capsys) + + assert "Explanation text here." in stdout + + +class TestProjectScopedKnowledgeWiring: + def test_project_flag_is_forwarded_to_registry_loader( + self, tmp_path: Path, capsys: pytest.CaptureFixture + ) -> None: + plan = _make_minimal_plan() + ctx_dir = tmp_path / ".claude" / "team-context" + ctx_dir.mkdir(parents=True) + project_root = tmp_path / "project-root" + project_root.mkdir() + args = _make_args(save=False, verbose=False) + args.project = str(project_root) + + mock_planner = MagicMock() + mock_planner.create_plan.return_value = plan + mock_registry = MagicMock() + + patches = [ + patch("agent_baton.cli.commands.execution.plan_cmd.IntelligentPlanner", return_value=mock_planner), + patch("agent_baton.cli.commands.execution.plan_cmd.KnowledgeRegistry", return_value=mock_registry), + patch("agent_baton.cli.commands.execution.plan_cmd.RetrospectiveEngine", return_value=MagicMock()), + patch("agent_baton.cli.commands.execution.plan_cmd.DataClassifier", return_value=MagicMock()), + patch("agent_baton.cli.commands.execution.plan_cmd.PolicyEngine", return_value=MagicMock()), + patch("agent_baton.core.orchestration.context.ContextManager", return_value=MagicMock()), + patch("agent_baton.cli.commands.execution.plan_cmd._persist_plan_to_db", MagicMock()), + patch.object( + Path, + "resolve", + lambda self: ctx_dir if self == Path(".claude/team-context") else Path(str(self)), + ), + ] + + with contextlib.ExitStack() as stack: + for p in patches: + stack.enter_context(p) + plan_cmd.handler(args) + + capsys.readouterr() + mock_registry.load_default_paths.assert_called_once_with( + project_root=project_root + ) diff --git a/tests/conftest.py b/tests/conftest.py index 0c7474a3..94c70f6e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -94,6 +94,24 @@ # Fixtures # --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def _sandbox_home( + tmp_path_factory: pytest.TempPathFactory, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Point HOME at a sandbox so tests never read the developer's ~/.claude. + + IntelligentPlanner (and the agent/knowledge registries behind it) eagerly + load ``~/.claude/knowledge`` and ``~/.claude/agents`` on construction; + without this, plan shapes depend on whatever packs the host machine has + installed. Tests that need a specific home layout set HOME/USERPROFILE + themselves, which overrides this fixture. + """ + home = tmp_path_factory.mktemp("sandbox-home") + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + + @pytest.fixture def sample_agent_content() -> str: """Raw string content of a valid agent .md file.""" diff --git a/tests/engine/planning/test_planner_diagnostics.py b/tests/engine/planning/test_planner_diagnostics.py new file mode 100644 index 00000000..e24b58ef --- /dev/null +++ b/tests/engine/planning/test_planner_diagnostics.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from agent_baton.core.engine.planning.planner import build_plan_diagnostics +from agent_baton.models.execution import MachinePlan, PlanPhase, PlanStep, TeamMember + + +def test_build_plan_diagnostics_preserves_existing_agents_and_includes_team_members() -> None: + plan = MachinePlan( + task_id="diag-team-task", + task_summary="Run a coordinated team step", + phases=[ + PlanPhase( + phase_id=0, + name="Implement", + steps=[ + PlanStep( + step_id="1.1", + agent_name="team", + task_description="Coordinate implementation", + team=[ + TeamMember( + member_id="1.1.a", + agent_name="backend-engineer", + role="lead", + task_description="Lead implementation", + sub_team=[ + TeamMember( + member_id="1.1.a.i", + agent_name="test-engineer", + role="implementer", + task_description="Add tests", + ) + ], + ), + TeamMember( + member_id="1.1.b", + agent_name="code-reviewer", + role="reviewer", + task_description="Review the implementation", + ), + ], + ) + ], + ) + ], + ) + plan.plan_diagnostics = { + "selected_agents": ["architect", "backend-engineer"], + } + + diagnostics = build_plan_diagnostics(plan) + + assert diagnostics["selected_agents"] == [ + "architect", + "backend-engineer", + "test-engineer", + "code-reviewer", + ] diff --git a/tests/engine/planning/test_validation_stage.py b/tests/engine/planning/test_validation_stage.py index db2ed843..99d1313d 100644 --- a/tests/engine/planning/test_validation_stage.py +++ b/tests/engine/planning/test_validation_stage.py @@ -10,10 +10,12 @@ 2. Detects defects independently of the reviewer: * empty_plan / empty_phase * agent_phase_mismatch (bd-0e36 / bd-1974 family) + * review_missing / audit_missing for missing quality coverage * review_skipped on non-light plans 3. Surfaces defects on ``draft.plan_defects``. - 4. Under ``BATON_PLANNER_HARD_GATE=1`` raises ``PlanQualityError`` - when any defect is critical. + 4. Raises ``PlanQualityError`` for critical defects by default; explicit + warn-only/dev mode downgrades to warnings unless the legacy hard-gate + override is truthy. """ from __future__ import annotations @@ -35,7 +37,7 @@ def _stub_services(planner: IntelligentPlanner) -> PlannerServices: """Build a minimal services container backed by *planner*.""" - return planner._build_services() + return planner._build_services(knowledge_registry=planner.knowledge_registry) class TestPlanDefect: @@ -97,7 +99,7 @@ def test_clean_plan_yields_no_critical_defects(self) -> None: assert len(plan.phases) >= 1 -class TestHardGate: +class TestGatePolicy: def teardown_method(self) -> None: os.environ.pop("BATON_PLANNER_HARD_GATE", None) @@ -125,8 +127,7 @@ def test_critical_defect_raises_under_hard_gate(self) -> None: stage.run(draft, services) assert "empty_plan" in str(ei.value) - def test_critical_defect_only_warns_without_hard_gate(self) -> None: - # No env var set. + def test_legacy_hard_gate_env_defaults_false(self) -> None: stage = ValidationStage() assert not stage._hard_gate_enabled() diff --git a/tests/engine/test_team_backends.py b/tests/engine/test_team_backends.py index ffa4fdd3..70af5931 100644 --- a/tests/engine/test_team_backends.py +++ b/tests/engine/test_team_backends.py @@ -8,19 +8,24 @@ from pathlib import Path import pytest +import yaml from agent_baton.core.engine.team_backends import ( ClaudeTeamsBackend, TeamBackend, WorktreeTeamBackend, audit_agents_for_teammate_safety, + build_team_readiness_diagnostics, check_resumability_constraints, + format_team_readiness_summary, select_team_backend, + write_team_readiness_report, ) from agent_baton.models.execution import ( MachinePlan, PlanPhase, PlanStep, + SynthesisSpec, TeamMember, ) @@ -67,10 +72,19 @@ def test_env_var_claude_teams(self, monkeypatch: pytest.MonkeyPatch) -> None: def test_unknown_value_falls_back( self, monkeypatch: pytest.MonkeyPatch, caplog, ) -> None: + monkeypatch.delenv("BATON_TEAMS_BACKEND_STRICT", raising=False) monkeypatch.setenv("BATON_TEAMS_BACKEND", "totally-made-up") be = select_team_backend() assert isinstance(be, WorktreeTeamBackend) + def test_unknown_value_raises_in_strict_mode( + self, monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("BATON_TEAMS_BACKEND", "totally-made-up") + monkeypatch.setenv("BATON_TEAMS_BACKEND_STRICT", "1") + with pytest.raises(ValueError, match="Unknown BATON_TEAMS_BACKEND"): + select_team_backend() + def test_explicit_arg_overrides_env( self, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -113,6 +127,7 @@ def test_writes_spawn_prompt(self, tmp_path: Path) -> None: # Known-limitation guidance is included so the lead reads it. assert "skills" in text and "mcpServers" in text assert "No in-process resumption" in text + assert "fixed permissions" in text.lower() def test_hook_command_includes_member_id(self) -> None: be = ClaudeTeamsBackend() @@ -150,6 +165,20 @@ def test_audit_flags_mcp_servers(self, tmp_path: Path) -> None: assert "mcp-agent" in flagged assert "mcpServers" in flagged["mcp-agent"] + def test_audit_flags_inline_list_values(self, tmp_path: Path) -> None: + agents = tmp_path / "agents" + agents.mkdir() + (agents / "inline-list-agent.md").write_text( + "---\n" + "name: inline-list-agent\n" + "model: sonnet\n" + "skills: [github]\n" + "---\n", + encoding="utf-8", + ) + flagged = audit_agents_for_teammate_safety(agents) + assert flagged == {"inline-list-agent": ["skills"]} + def test_empty_lists_are_not_flagged(self, tmp_path: Path) -> None: agents = tmp_path / "agents" agents.mkdir() @@ -160,6 +189,64 @@ def test_empty_lists_are_not_flagged(self, tmp_path: Path) -> None: flagged = audit_agents_for_teammate_safety(agents) assert flagged == {} + def test_comment_only_keys_are_not_flagged(self, tmp_path: Path) -> None: + agents = tmp_path / "agents" + agents.mkdir() + (agents / "comment-only.md").write_text( + "---\n" + "name: comment-only\n" + "model: sonnet\n" + "skills: # none\n" + "mcpServers: # none\n" + "---\n", + encoding="utf-8", + ) + flagged = audit_agents_for_teammate_safety(agents) + assert flagged == {} + + def test_audit_flags_yaml_block_forms(self, tmp_path: Path) -> None: + agents = tmp_path / "agents" + agents.mkdir() + (agents / "block-agent.md").write_text( + "---\n" + "name: block-agent\n" + "model: sonnet\n" + "skills:\n" + " - github\n" + "mcpServers:\n" + " filesystem:\n" + " command: npx\n" + "---\n" + "body\n", + encoding="utf-8", + ) + flagged = audit_agents_for_teammate_safety(agents) + assert flagged == {"block-agent": ["skills", "mcpServers"]} + + def test_audit_flags_pyyaml_safe_dump_zero_indent_block_sequence( + self, tmp_path: Path + ) -> None: + agents = tmp_path / "agents" + agents.mkdir() + frontmatter = yaml.safe_dump( + { + "name": "safe-dump-agent", + "model": "sonnet", + "skills": ["github"], + "mcpServers": {"filesystem": {"command": "npx"}}, + }, + sort_keys=False, + ) + assert "\n- github\n" in frontmatter + (agents / "safe-dump-agent.md").write_text( + f"---\n{frontmatter}---\nbody\n", + encoding="utf-8", + ) + + flagged = audit_agents_for_teammate_safety(agents) + + assert flagged == {"safe-dump-agent": ["skills", "mcpServers"]} + def test_audit_skips_non_frontmatter_files(self, tmp_path: Path) -> None: agents = tmp_path / "agents" agents.mkdir() @@ -172,6 +259,94 @@ def test_audit_missing_dir_returns_empty(self, tmp_path: Path) -> None: assert flagged == {} +class TestTeamReadinessDiagnostics: + def test_worktree_diagnostics_summarize_team_shape(self, tmp_path: Path) -> None: + plan = _plan_with_team() + step = plan.phases[0].steps[0] + step.context_files = ["README.md", "docs/engine-and-runtime.md"] + step.synthesis = SynthesisSpec( + strategy="merge_files", + conflict_handling="escalate", + ) + + diagnostics = build_team_readiness_diagnostics( + plan=plan, + step=step, + backend_name="worktree", + team_context_root=tmp_path, + ).to_dict() + + assert diagnostics["backend"] == "worktree" + assert diagnostics["member_count"] == 2 + assert diagnostics["nested_team_count"] == 0 + assert diagnostics["shared_files"] == [ + "README.md", + "docs/engine-and-runtime.md", + ] + assert diagnostics["shared_contracts"][0]["member_id"] == "1.1.a" + assert diagnostics["synthesis_strategy"] == "merge_files" + assert diagnostics["conflict_strategy"] == "escalate" + assert diagnostics["warning_count"] == 0 + + def test_claude_teams_diagnostics_surface_caveats( + self, tmp_path: Path, + ) -> None: + plan = TestSpawnPromptFidelity()._nested_plan() + diagnostics = build_team_readiness_diagnostics( + plan=plan, + step=plan.phases[0].steps[0], + backend_name="claude-teams", + team_context_root=tmp_path, + ).to_dict() + + assert diagnostics["backend"] == "claude-teams" + assert diagnostics["member_count"] == 2 + assert diagnostics["nested_team_count"] == 1 + warnings = "\n".join(diagnostics["warnings"]).lower() + assert "no resume" in warnings or "cannot resume" in warnings + assert "no nesting" in warnings or "cannot nest" in warnings + assert "one team at a time" in warnings + assert "fixed permissions" in warnings + assert "skills/mcp" in warnings or "mcpservers" in warnings + assert diagnostics["warning_count"] >= 5 + + def test_step_specific_warnings_precede_static_caveats( + self, tmp_path: Path, + ) -> None: + plan = TestSpawnPromptFidelity()._nested_plan() + diagnostics = build_team_readiness_diagnostics( + plan=plan, + step=plan.phases[0].steps[0], + backend_name="claude-teams", + team_context_root=tmp_path, + ) + + # The dispatch summary caps warning_notes; actionable step-specific + # warnings must come before the static claude-teams caveats. + assert "nested team" in diagnostics.warnings[0] + assert "nested team" in format_team_readiness_summary(diagnostics) + + def test_report_path_is_posix_relative(self, tmp_path: Path) -> None: + plan = _plan_with_team() + step = plan.phases[0].steps[0] + diagnostics = build_team_readiness_diagnostics( + plan=plan, + step=step, + backend_name="worktree", + team_context_root=tmp_path, + ) + + written = write_team_readiness_report( + diagnostics=diagnostics, + team_context_root=tmp_path, + ) + + assert written.report_path == ( + f"teams/team-{step.step_id}/team-report.json" + ) + assert "\\" not in written.report_path + + class TestSpawnPromptFidelity: """A1.a/A1.b/A1.c: spawn.md flattening, safety guards, size + approval.""" diff --git a/tests/engine/test_team_mailbox_hooks.py b/tests/engine/test_team_mailbox_hooks.py index dd2a11a5..a88a19ab 100644 --- a/tests/engine/test_team_mailbox_hooks.py +++ b/tests/engine/test_team_mailbox_hooks.py @@ -6,12 +6,14 @@ """ from __future__ import annotations +import json from pathlib import Path import pytest from agent_baton.core.engine.executor import ExecutionEngine from agent_baton.core.engine.mailbox import TeamMailbox +from agent_baton.core.engine.persistence import StatePersistence from agent_baton.models.execution import ( MachinePlan, PlanPhase, @@ -49,6 +51,49 @@ def _team_plan() -> MachinePlan: ) +def _solo_then_team_plan() -> MachinePlan: + return MachinePlan( + task_id="t-next-actions-team", + task_summary="unlock team from next_actions", + phases=[ + PlanPhase( + phase_id=1, + name="Build", + steps=[ + PlanStep( + step_id="1.1", + agent_name="setup-agent", + task_description="prepare inputs", + ), + PlanStep( + step_id="1.2", + agent_name="team", + task_description="implement and review", + depends_on=["1.1"], + model="sonnet", + team=[ + TeamMember( + member_id="1.2.a", + agent_name="backend-engineer", + role="implementer", + task_description="write the service", + model="sonnet", + ), + TeamMember( + member_id="1.2.b", + agent_name="code-reviewer", + role="reviewer", + task_description="review for security", + model="sonnet", + ), + ], + ), + ], + ), + ], + ) + + def _started(tmp_path: Path) -> tuple[ExecutionEngine, MachinePlan]: engine = ExecutionEngine(team_context_root=tmp_path) plan = _team_plan() @@ -80,6 +125,73 @@ def test_task_created_carries_payload(self, tmp_path: Path) -> None: assert created.payload["role"] == "implementer" assert created.payload["step_id"] == "1.1" + def test_team_report_written_with_readiness_diagnostics( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("BATON_TEAMS_BACKEND", raising=False) + monkeypatch.delenv("BATON_TEAMS_BACKEND_STRICT", raising=False) + + engine = ExecutionEngine(team_context_root=tmp_path) + action = engine.start(_team_plan()) + + report = tmp_path / "teams" / "team-1.1" / "team-report.json" + assert report.exists() + payload = json.loads(report.read_text(encoding="utf-8")) + assert payload["backend"] == "worktree" + assert payload["member_count"] == 2 + assert payload["nested_team_count"] == 0 + assert payload["synthesis_strategy"] == "concatenate" + assert payload["conflict_strategy"] == "auto_merge" + assert payload["warning_count"] == 0 + assert "Team readiness: backend=worktree" in action.message + + state = StatePersistence(tmp_path).load() + assert state is not None + assert ( + state.plan.plan_diagnostics["team_readiness"]["1.1"]["backend"] + == "worktree" + ) + + def test_strict_unknown_backend_blocks_team_dispatch( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("BATON_TEAMS_BACKEND", "not-real") + monkeypatch.setenv("BATON_TEAMS_BACKEND_STRICT", "1") + engine = ExecutionEngine(team_context_root=tmp_path) + + with pytest.raises(ValueError, match="Unknown BATON_TEAMS_BACKEND"): + engine.start(_team_plan()) + + def test_next_actions_persists_team_readiness_for_unlocked_team( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("BATON_TEAMS_BACKEND", raising=False) + monkeypatch.delenv("BATON_TEAMS_BACKEND_STRICT", raising=False) + engine = ExecutionEngine(team_context_root=tmp_path) + engine.start(_solo_then_team_plan()) + engine.record_step_result( + step_id="1.1", + agent_name="setup-agent", + status="complete", + outcome="inputs ready", + ) + + actions = engine.next_actions() + + assert actions + state = StatePersistence(tmp_path).load() + assert state is not None + assert ( + state.plan.plan_diagnostics["team_readiness"]["1.2"]["backend"] + == "worktree" + ) + class TestMailboxOnMemberResult: def test_task_completed_event_on_success(self, tmp_path: Path) -> None: diff --git a/tests/knowledge/test_knowledge_doctor.py b/tests/knowledge/test_knowledge_doctor.py new file mode 100644 index 00000000..c434442e --- /dev/null +++ b/tests/knowledge/test_knowledge_doctor.py @@ -0,0 +1,364 @@ +"""Tests for ``baton knowledge doctor``.""" +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + + +def _run_cli(argv: list[str]) -> int: + from agent_baton.cli.main import main + + try: + main(argv) + return 0 + except SystemExit as exc: + return int(exc.code) if exc.code is not None else 0 + + +def _isolate_defaults(monkeypatch, tmp_path: Path) -> None: + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.chdir(tmp_path) + + +def _write_doc( + path: Path, + *, + name: str | None = None, + description: str | None = None, + tags: list[str] | None = None, + body: str = "body", +) -> None: + metadata: dict[str, object] = {} + if name is not None: + metadata["name"] = name + if description is not None: + metadata["description"] = description + if tags is not None: + metadata["tags"] = tags + + if metadata: + text = "---\n" + yaml.safe_dump(metadata, sort_keys=False) + "---\n" + body + else: + text = body + path.write_text(text, encoding="utf-8") + + +def test_doctor_json_reports_actionable_pack_warnings( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + knowledge = tmp_path / ".claude" / "knowledge" + + missing_manifest = knowledge / "missing-manifest" + missing_manifest.mkdir(parents=True) + _write_doc( + missing_manifest / "guide.md", + name="guide", + description="Valid guide", + tags=["auth"], + ) + + broken = knowledge / "broken" + broken.mkdir() + (broken / "knowledge.yaml").write_text( + yaml.safe_dump( + { + "name": "broken", + "description": "Broken fixture", + "default_delivery": "inline", + "documents": [{"path": "missing.md"}], + }, + sort_keys=False, + ), + encoding="utf-8", + ) + _write_doc(broken / "a.md", name="duplicate", description="", body="short") + _write_doc( + broken / "b.md", + name="duplicate", + description="Second duplicate", + body="short", + ) + _write_doc(broken / "raw.md", body="no frontmatter") + _write_doc( + broken / "large.md", + name="large-inline", + description="Large inline candidate", + tags=["auth"], + body="x" * 3000, + ) + + rc = _run_cli(["knowledge", "doctor", "--json"]) + out = capsys.readouterr().out + data = json.loads(out) + codes = {issue["code"] for issue in data["issues"]} + + assert rc == 0 + assert data["ok"] is False + assert "missing-manifest" in codes + assert "empty-doc-description" in codes + assert "duplicate-doc-name" in codes + assert "missing-declared-file" in codes + assert "empty-doc-metadata" in codes + assert "large-inline-candidate" in codes + assert all("Edit " in issue["message"] for issue in data["issues"]) + + +def test_doctor_strict_exits_nonzero_for_warnings( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + pack = tmp_path / ".claude" / "knowledge" / "missing-manifest" + pack.mkdir(parents=True) + _write_doc(pack / "guide.md", name="guide", description="Valid guide") + + rc = _run_cli(["knowledge", "doctor", "--strict"]) + out = capsys.readouterr().out + + assert rc == 1 + assert "missing-manifest" in out + assert "Edit " in out + + +def test_doctor_clean_pack_reports_ok(tmp_path: Path, monkeypatch, capsys) -> None: + _isolate_defaults(monkeypatch, tmp_path) + pack = tmp_path / ".claude" / "knowledge" / "clean" + pack.mkdir(parents=True) + (pack / "knowledge.yaml").write_text( + yaml.safe_dump( + { + "name": "clean", + "description": "Clean pack", + "tags": ["auth"], + "default_delivery": "reference", + }, + sort_keys=False, + ), + encoding="utf-8", + ) + _write_doc( + pack / "guide.md", + name="auth-guide", + description="Authentication guide", + tags=["auth"], + ) + + rc = _run_cli(["knowledge", "doctor", "--json"]) + data = json.loads(capsys.readouterr().out) + + assert rc == 0 + assert data["ok"] is True + assert data["issues"] == [] + assert data["summary"]["documents"] == 1 + + +def test_doctor_accepts_docs_stems_and_reports_missing_stems( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + pack = tmp_path / ".claude" / "knowledge" / "harvested" + pack.mkdir(parents=True) + (pack / "knowledge.yaml").write_text( + yaml.safe_dump( + { + "name": "harvested", + "description": "Harvested pack", + "tags": ["taxonomy"], + "default_delivery": "reference", + "docs": ["taxonomy-quick-ref", "missing-stem"], + }, + sort_keys=False, + ), + encoding="utf-8", + ) + _write_doc( + pack / "taxonomy-quick-ref.md", + name="taxonomy-quick-ref", + description="Taxonomy quick reference", + tags=["taxonomy"], + ) + + rc = _run_cli(["knowledge", "doctor", "--json"]) + data = json.loads(capsys.readouterr().out) + missing = [ + issue + for issue in data["issues"] + if issue["code"] == "missing-declared-file" + ] + + assert rc == 0 + assert len(missing) == 1 + assert "missing-stem" in missing[0]["message"] + assert "taxonomy-quick-ref" not in missing[0]["message"] + + +def test_doctor_reports_invalid_doc_frontmatter_for_sequence_and_scalar( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + pack = tmp_path / ".claude" / "knowledge" / "invalid-frontmatter" + pack.mkdir(parents=True) + (pack / "knowledge.yaml").write_text( + yaml.safe_dump( + { + "name": "invalid-frontmatter", + "description": "Invalid frontmatter fixtures", + "default_delivery": "reference", + }, + sort_keys=False, + ), + encoding="utf-8", + ) + (pack / "sequence.md").write_text( + "---\n- not\n- a\n- mapping\n---\nbody\n", encoding="utf-8" + ) + (pack / "scalar.md").write_text( + "---\njust-a-string\n---\nbody\n", encoding="utf-8" + ) + + rc = _run_cli(["knowledge", "doctor", "--json"]) + data = json.loads(capsys.readouterr().out) + invalid = [ + issue + for issue in data["issues"] + if issue["code"] == "invalid-doc-frontmatter" + ] + + assert rc == 0 + assert {issue["doc"] for issue in invalid} == {"sequence", "scalar"} + assert all("Edit " in issue["message"] for issue in invalid) + assert not any( + issue["code"] == "empty-doc-description" + for issue in data["issues"] + ) + + +def test_doctor_reports_non_utf8_manifest_as_invalid_manifest( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + pack = tmp_path / ".claude" / "knowledge" / "legacy-manifest" + pack.mkdir(parents=True) + (pack / "knowledge.yaml").write_bytes( + b"name: legacy-\xe9\ndescription: legacy manifest\n" + ) + _write_doc( + pack / "guide.md", + name="guide", + description="Valid guide", + tags=["legacy"], + ) + + rc = _run_cli(["knowledge", "doctor", "--json"]) + data = json.loads(capsys.readouterr().out) + invalid_manifest = [ + issue + for issue in data["issues"] + if issue["code"] == "invalid-manifest" + ] + + assert rc == 0 + assert len(invalid_manifest) == 1 + assert invalid_manifest[0]["pack"] == "legacy-manifest" + assert "Edit " in invalid_manifest[0]["message"] + + +def test_doctor_reports_non_utf8_doc_as_unreadable_doc( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + pack = tmp_path / ".claude" / "knowledge" / "legacy-doc" + pack.mkdir(parents=True) + (pack / "knowledge.yaml").write_text( + yaml.safe_dump( + { + "name": "legacy-doc", + "description": "Legacy encoded document", + "default_delivery": "reference", + }, + sort_keys=False, + ), + encoding="utf-8", + ) + (pack / "legacy.md").write_bytes( + b"---\nname: legacy\ndescription: Caf\xe9\n---\nbody\n" + ) + + rc = _run_cli(["knowledge", "doctor", "--json"]) + data = json.loads(capsys.readouterr().out) + unreadable = [ + issue + for issue in data["issues"] + if issue["code"] == "unreadable-doc" + ] + + assert rc == 0 + assert len(unreadable) == 1 + assert unreadable[0]["doc"] == "legacy" + assert "Edit " in unreadable[0]["message"] + assert not any( + issue["code"] == "empty-doc-description" + for issue in data["issues"] + ) + + +def test_declared_doc_with_dotted_stem_resolves_md_fallback( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + pack = tmp_path / ".claude" / "knowledge" / "dotted-stem" + pack.mkdir(parents=True) + (pack / "knowledge.yaml").write_text( + yaml.safe_dump( + { + "name": "dotted-stem", + "description": "Dotted stem fixtures", + "documents": ["notes.v2"], + }, + sort_keys=False, + ), + encoding="utf-8", + ) + _write_doc( + pack / "notes.v2.md", + name="notes.v2", + description="Versioned notes", + tags=["versioned"], + ) + + rc = _run_cli(["knowledge", "doctor", "--json"]) + data = json.loads(capsys.readouterr().out) + + assert rc == 0 + assert not any( + issue["code"] == "missing-declared-file" for issue in data["issues"] + ) + + +def test_strict_missing_explicit_root_emits_issue_and_fails( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + missing_root = tmp_path / "does-not-exist" + + rc = _run_cli([ + "knowledge", + "doctor", + "--json", + "--knowledge-root", + str(missing_root), + "--strict", + ]) + data = json.loads(capsys.readouterr().out) + missing = [ + issue for issue in data["issues"] if issue["code"] == "missing-root" + ] + + assert rc == 1 + assert len(missing) == 1 + assert "does-not-exist" in missing[0]["message"] diff --git a/tests/knowledge/test_knowledge_parser_contract.py b/tests/knowledge/test_knowledge_parser_contract.py new file mode 100644 index 00000000..95eabb8c --- /dev/null +++ b/tests/knowledge/test_knowledge_parser_contract.py @@ -0,0 +1,137 @@ +"""Parser contract tests for the shared ``baton knowledge`` command tree.""" +from __future__ import annotations + +import argparse + +from agent_baton.cli.main import discover_commands + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="baton") + subparsers = parser.add_subparsers(dest="command") + for mod in discover_commands().values(): + mod.register(subparsers) + return parser + + +def _knowledge_subparser( + parser: argparse.ArgumentParser, +) -> argparse._SubParsersAction: # type: ignore[type-arg] + top_level = next( + action + for action in parser._actions + if isinstance(action, argparse._SubParsersAction) + ) + knowledge = top_level.choices["knowledge"] + nested = [ + action + for action in knowledge._actions + if isinstance(action, argparse._SubParsersAction) + ] + assert len(nested) == 1 + return nested[0] + + +def test_knowledge_parent_uses_shared_dest_and_all_subcommands() -> None: + parser = _build_parser() + sub = _knowledge_subparser(parser) + + assert sub.dest == "knowledge_cmd" + assert { + "ab", + "brief", + "doctor", + "effectiveness", + "harvest", + "ranking", + "resolve", + "search", + "usage", + "stale", + "sweep", + "deprecate", + "retire", + } <= set(sub.choices) + + +def test_parse_knowledge_ab_keeps_nested_ab_dest() -> None: + parser = _build_parser() + + args = parser.parse_args(["knowledge", "ab", "list"]) + + assert args.command == "knowledge" + assert args.knowledge_cmd == "ab" + assert args.ab_subcommand == "list" + + +def test_parse_knowledge_doctor_contract() -> None: + parser = _build_parser() + + args = parser.parse_args([ + "knowledge", + "doctor", + "--knowledge-root", + "X", + "--format", + "json", + "--strict", + ]) + + assert args.knowledge_cmd == "doctor" + assert args.knowledge_root == ["X"] + assert args.format == "json" + assert args.strict is True + + +def test_parse_knowledge_search_contract() -> None: + parser = _build_parser() + + args = parser.parse_args([ + "knowledge", + "search", + "auth middleware", + "--knowledge-root", + "X", + "--limit", + "5", + "--format", + "json", + ]) + + assert args.knowledge_cmd == "search" + assert args.query == ["auth middleware"] + assert args.knowledge_root == ["X"] + assert args.limit == 5 + assert args.format == "json" + + +def test_parse_knowledge_resolve_contract() -> None: + parser = _build_parser() + + args = parser.parse_args([ + "knowledge", + "resolve", + "--agent", + "backend-engineer--python", + "--task", + "Fix auth middleware", + "--task-type", + "bug-fix", + "--risk", + "HIGH", + "--knowledge-pack", + "security", + "--knowledge", + "docs/auth.md", + "--format", + "json", + ]) + + assert args.knowledge_cmd == "resolve" + assert args.agent == "backend-engineer--python" + assert args.task == "Fix auth middleware" + assert args.task_type == "bug-fix" + assert args.risk == "HIGH" + assert args.knowledge_pack == ["security"] + assert args.knowledge == ["docs/auth.md"] + assert args.format == "json" diff --git a/tests/knowledge/test_knowledge_search.py b/tests/knowledge/test_knowledge_search.py new file mode 100644 index 00000000..e46ae020 --- /dev/null +++ b/tests/knowledge/test_knowledge_search.py @@ -0,0 +1,139 @@ +"""Tests for knowledge search and resolve simulation CLI commands.""" +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +from agent_baton.core.engine.knowledge_resolver import KnowledgeResolver +from agent_baton.core.orchestration.knowledge_registry import KnowledgeRegistry +from agent_baton.core.orchestration.registry import AgentRegistry + + +def _run_cli(argv: list[str]) -> int: + from agent_baton.cli.main import main + + try: + main(argv) + return 0 + except SystemExit as exc: + return int(exc.code) if exc.code is not None else 0 + + +def _isolate_defaults(monkeypatch, tmp_path: Path) -> None: + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.chdir(tmp_path) + + +def _write_pack(tmp_path: Path) -> tuple[Path, Path]: + knowledge = tmp_path / ".claude" / "knowledge" + pack = knowledge / "auth-pack" + pack.mkdir(parents=True) + (pack / "knowledge.yaml").write_text( + yaml.safe_dump( + { + "name": "auth-pack", + "description": "Authentication token renewal and session rules", + "tags": ["authentication", "tokens"], + "target_agents": ["backend-engineer--python"], + "default_delivery": "reference", + }, + sort_keys=False, + ), + encoding="utf-8", + ) + doc = pack / "renewal.md" + doc.write_text( + "---\n" + "name: token-renewal\n" + "description: Authentication token renewal flow and refresh handling\n" + "tags: [authentication, token, renewal]\n" + "priority: high\n" + "---\n" + "Renew short-lived authentication tokens before expiry.\n", + encoding="utf-8", + ) + return pack, doc + + +def _write_agent(tmp_path: Path) -> None: + agents = tmp_path / ".claude" / "agents" + agents.mkdir(parents=True) + (agents / "backend-engineer--python.md").write_text( + "---\n" + "name: backend-engineer--python\n" + "description: Backend engineer\n" + "knowledge_packs: [auth-pack]\n" + "---\n" + "# Agent\n", + encoding="utf-8", + ) + + +def test_search_json_returns_registry_metadata( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + _pack, doc = _write_pack(tmp_path) + + rc = _run_cli([ + "knowledge", + "search", + "authentication token renewal", + "--json", + ]) + data = json.loads(capsys.readouterr().out) + + assert rc == 0 + assert data["query"] == "authentication token renewal" + assert data["results"] + first = data["results"][0] + assert first["pack"] == "auth-pack" + assert first["doc"] == "token-renewal" + assert first["score"] > 0 + assert first["path"] == str(doc) + assert first["tags"] == ["authentication", "token", "renewal"] + assert first["priority"] == "high" + assert first["token_estimate"] > 0 + + +def test_resolve_json_matches_actual_resolver_output( + tmp_path: Path, monkeypatch, capsys +) -> None: + _isolate_defaults(monkeypatch, tmp_path) + _write_pack(tmp_path) + _write_agent(tmp_path) + + rc = _run_cli([ + "knowledge", + "resolve", + "--agent", + "backend-engineer--python", + "--task", + "refresh authentication token renewal", + "--json", + ]) + cli_data = json.loads(capsys.readouterr().out) + + registry = KnowledgeRegistry() + registry.load_default_paths() + agent_registry = AgentRegistry() + agent_registry.load_default_paths() + expected = [ + attachment.to_dict() + for attachment in KnowledgeResolver( + registry, agent_registry=agent_registry + ).resolve( + agent_name="backend-engineer--python", + task_description="refresh authentication token renewal", + ) + ] + + assert rc == 0 + assert cli_data["attachments"] == expected + assert cli_data["attachments"][0]["pack_name"] == "auth-pack" + assert cli_data["attachments"][0]["document_name"] == "token-renewal" diff --git a/tests/models/test_execution_sqlite_roundtrip.py b/tests/models/test_execution_sqlite_roundtrip.py index c9428a45..e178a90f 100644 --- a/tests/models/test_execution_sqlite_roundtrip.py +++ b/tests/models/test_execution_sqlite_roundtrip.py @@ -337,6 +337,19 @@ def test_plan_sqlite_roundtrip_field_parity(self, store: SqliteStorage) -> None: assert loaded.task_type == plan.task_type assert loaded.intervention_level == plan.intervention_level + def test_plan_sqlite_plan_diagnostics_roundtrip(self, store: SqliteStorage) -> None: + """Non-empty plan diagnostics survive save_plan -> load_plan.""" + plan = _minimal_plan("task-plan-rt-006") + plan.plan_diagnostics = { + "classification_source": "headless-claude", + "knowledge_packs_loaded": 2, + "attachments_selected": 1, + } + store.save_plan(plan) + loaded = store.load_plan("task-plan-rt-006") + assert loaded is not None + assert loaded.plan_diagnostics == plan.plan_diagnostics + def test_plan_sqlite_phases_and_steps_preserved(self, store: SqliteStorage) -> None: """Phase and step hierarchy survives save_plan → load_plan.""" plan = _minimal_plan("task-plan-rt-002") @@ -609,6 +622,22 @@ def test_execution_sqlite_plan_preserved(self, store: SqliteStorage) -> None: assert loaded.plan.task_id == plan.task_id assert len(loaded.plan.phases) == len(plan.phases) + def test_execution_sqlite_plan_diagnostics_preserved( + self, store: SqliteStorage + ) -> None: + """Embedded MachinePlan diagnostics survive save_execution -> load_execution.""" + plan = _minimal_plan("task-exec-rt-009b") + plan.plan_diagnostics = { + "classification_source": "headless-claude", + "knowledge_packs_loaded": 2, + "attachments_selected": 1, + } + state = _minimal_execution_state(plan) + store.save_execution(state) + loaded = store.load_execution("task-exec-rt-009b") + assert loaded is not None + assert loaded.plan.plan_diagnostics == plan.plan_diagnostics + def test_execution_sqlite_load_returns_none_for_missing( self, store: SqliteStorage ) -> None: diff --git a/tests/planning/test_plan_quality_golden.py b/tests/planning/test_plan_quality_golden.py new file mode 100644 index 00000000..733288c0 --- /dev/null +++ b/tests/planning/test_plan_quality_golden.py @@ -0,0 +1,123 @@ +"""Snapshot-backed golden coverage for representative planner outputs.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from agent_baton.core.engine.planner import IntelligentPlanner + + +SNAPSHOT_DIR = Path(__file__).resolve().parents[1] / "snapshots" / "plans" + + +def _agent_base(agent_name: str) -> str: + return (agent_name or "").split("--")[0] + + +def _normalize_member(member: Any) -> dict[str, Any]: + data: dict[str, Any] = { + "member_id": member.member_id, + "agent": _agent_base(member.agent_name), + "role": member.role, + } + if member.depends_on: + data["depends_on"] = list(member.depends_on) + if member.sub_team: + data["sub_team"] = [_normalize_member(m) for m in member.sub_team] + return data + + +def _normalize_plan(plan: Any) -> dict[str, Any]: + return { + "task_type": plan.task_type, + "complexity": plan.complexity, + "risk_level": plan.risk_level, + "budget_tier": plan.budget_tier, + "phases": [ + { + "phase_id": phase.phase_id, + "name": phase.name, + "approval_required": phase.approval_required, + "steps": [ + { + "step_id": step.step_id, + "agent": _agent_base(step.agent_name), + "team": [_normalize_member(m) for m in step.team], + "depends_on": list(step.depends_on), + "step_type": step.step_type, + } + for step in phase.steps + ], + } + for phase in plan.phases + ], + } + + +def _load_snapshot(case_id: str) -> dict[str, Any]: + path = SNAPSHOT_DIR / f"{case_id}.json" + assert path.exists(), f"Missing golden plan snapshot: {path}" + return json.loads(path.read_text(encoding="utf-8")) + + +GOLDEN_CASES = [ + pytest.param( + "direct-light", + "Fix typo in README copy", + {"complexity": "light"}, + id="direct-light", + ), + pytest.param( + "investigative-bug", + "Investigate intermittent timeout in API requests and identify the likely root cause", + {"task_type": "bugfix"}, + id="investigative-bug", + ), + pytest.param( + "compound-multi-concern", + "Update the backend API, frontend dashboard, and test coverage for account settings", + {}, + id="compound-multi-concern", + ), + pytest.param( + "high-risk-security", + "Refactor authentication and payment authorization logic", + {}, + id="high-risk-security", + ), + pytest.param( + "compliance-audit", + "Ensure GDPR compliance for the user data export workflow", + {}, + id="compliance-audit", + ), + pytest.param( + "knowledge-heavy", + "Use project security knowledge to update JWT authentication guidance and tests", + {"explicit_knowledge_packs": ["security-pack"]}, + id="knowledge-heavy", + ), +] + + +@pytest.fixture(autouse=True) +def _clear_planner_gate_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Golden shapes assume default gate behavior; strip ambient overrides.""" + for var in ( + "BATON_DEV_MODE", + "BATON_PLANNER_WARN_ONLY", + "BATON_PLANNER_HARD_GATE", + ): + monkeypatch.delenv(var, raising=False) + + +@pytest.mark.parametrize(("case_id", "prompt", "kwargs"), GOLDEN_CASES) +def test_representative_plan_shapes_match_golden_snapshots( + case_id: str, prompt: str, kwargs: dict[str, Any] +) -> None: + plan = IntelligentPlanner().create_plan(prompt, **kwargs) + + assert _normalize_plan(plan) == _load_snapshot(case_id) diff --git a/tests/planning/test_plan_quality_validation.py b/tests/planning/test_plan_quality_validation.py new file mode 100644 index 00000000..19d16fd0 --- /dev/null +++ b/tests/planning/test_plan_quality_validation.py @@ -0,0 +1,328 @@ +"""Plan-quality validation gates for actionable planner defects.""" +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from agent_baton.core.engine.planner import IntelligentPlanner +from agent_baton.core.engine.planning.draft import PlanDraft +from agent_baton.core.engine.planning.services import PlannerServices +from agent_baton.core.engine.planning.stages.validation import ( + PlanDefect, + PlanQualityError, + ValidationStage, +) +from agent_baton.core.govern.classifier import ClassificationResult, DataClassifier +from agent_baton.models.enums import RiskLevel +from agent_baton.models.execution import PlanPhase, PlanStep + + +def _stub_services() -> PlannerServices: + planner = IntelligentPlanner() + return planner._build_services(knowledge_registry=planner.knowledge_registry) + + +def _draft_with_phase( + *, + task_summary: str = "Ship an auth change", + risk: RiskLevel = RiskLevel.LOW, + phase_name: str = "Implement", + agent_name: str = "backend-engineer", +) -> PlanDraft: + draft = PlanDraft.from_inputs(task_summary) + draft.task_id = "task-plan-quality" + draft.risk_level_enum = risk + draft.risk_level = risk.value + draft.inferred_complexity = "medium" + draft.resolved_agents = [agent_name] + draft.plan_phases = [ + PlanPhase( + phase_id=1, + name=phase_name, + steps=[ + PlanStep( + step_id="1.1", + agent_name=agent_name, + task_description="Implement the requested change.", + ) + ], + ) + ] + draft.review_result = None + return draft + + +def _run_stage_with_critical_defect(stage: ValidationStage, draft: PlanDraft) -> None: + with patch.object(stage, "_detect_defects") as detect: + detect.return_value = [ + PlanDefect( + code="empty_plan", + severity="critical", + message=( + "task_id=task-plan-quality phase_count=0. " + "Remediation: add phases." + ), + ) + ] + with patch.object(stage, "_check_scores", return_value="standard"): + with patch.object(stage, "_consolidate_team", return_value=([], None)): + stage.run(draft, _stub_services()) + + +class TestDefaultGatePolicy: + def test_critical_defect_blocks_by_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("BATON_PLANNER_HARD_GATE", raising=False) + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + stage = ValidationStage() + draft = _draft_with_phase() + + with pytest.raises(PlanQualityError) as ei: + _run_stage_with_critical_defect(stage, draft) + + assert "empty_plan" in str(ei.value) + assert "Remediation:" in str(ei.value) + + def test_hard_gate_zero_does_not_disable_default_blocking( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("BATON_PLANNER_HARD_GATE", "0") + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + stage = ValidationStage() + draft = _draft_with_phase() + + with pytest.raises(PlanQualityError): + _run_stage_with_critical_defect(stage, draft) + + @pytest.mark.parametrize( + ("env_name", "env_value"), + [("BATON_DEV_MODE", "1"), ("BATON_PLANNER_WARN_ONLY", "1")], + ) + def test_explicit_warn_only_modes_allow_critical_defects_to_warn( + self, + monkeypatch: pytest.MonkeyPatch, + env_name: str, + env_value: str, + ) -> None: + monkeypatch.delenv("BATON_PLANNER_HARD_GATE", raising=False) + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + monkeypatch.setenv(env_name, env_value) + stage = ValidationStage() + draft = _draft_with_phase() + + _run_stage_with_critical_defect(stage, draft) + + assert any("empty_plan" in warning for warning in draft.score_warnings) + + def test_legacy_hard_gate_overrides_dev_mode( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("BATON_PLANNER_HARD_GATE", "1") + monkeypatch.setenv("BATON_DEV_MODE", "1") + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + stage = ValidationStage() + draft = _draft_with_phase() + + with pytest.raises(PlanQualityError): + _run_stage_with_critical_defect(stage, draft) + + +class TestActionableDefectMessages: + def test_empty_plan_message_includes_context_and_remediation(self) -> None: + draft = PlanDraft.from_inputs("Add foo") + draft.task_id = "task-empty" + draft.plan_phases = [] + draft.review_result = None + + defects = ValidationStage()._detect_defects(draft) + message = next(d.message for d in defects if d.code == "empty_plan") + + assert "task-empty" in message + assert "phase_count=0" in message + assert "Remediation:" in message + assert "phase" in message.lower() + assert "step" in message.lower() + + def test_empty_phase_message_includes_phase_context_and_remediation(self) -> None: + draft = PlanDraft.from_inputs("Add foo") + draft.task_id = "task-empty-phase" + draft.plan_phases = [PlanPhase(phase_id=3, name="Implement", steps=[])] + draft.review_result = None + + defects = ValidationStage()._detect_defects(draft) + message = next(d.message for d in defects if d.code == "empty_phase") + + assert "phase_id=3" in message + assert "Implement" in message + assert "step_count=0" in message + assert "Remediation:" in message + + def test_agent_phase_mismatch_message_includes_step_phase_and_remediation(self) -> None: + draft = _draft_with_phase(agent_name="architect") + + defects = ValidationStage()._detect_defects(draft) + message = next(d.message for d in defects if d.code == "agent_phase_mismatch") + + assert "phase_id=1" in message + assert "step_id=1.1" in message + assert "architect" in message + assert "Implement" in message + assert "Remediation:" in message + + def test_review_skipped_message_includes_review_context_and_remediation(self) -> None: + draft = _draft_with_phase() + draft.inferred_complexity = "heavy" + draft.review_result = SimpleNamespace(source="skipped-light", warnings=[]) + + defects = ValidationStage()._detect_defects(draft) + message = next(d.message for d in defects if d.code == "review_skipped") + + assert "task-plan-quality" in message + assert "source=skipped-light" in message + assert "complexity=heavy" in message + assert "Remediation:" in message + + def test_reviewer_critical_warning_appends_remediation_when_absent(self) -> None: + draft = _draft_with_phase() + draft.review_result = SimpleNamespace( + source="reviewed", + warnings=["[critical] Review phase missing for high-risk plan"], + ) + + defects = ValidationStage()._detect_defects(draft) + message = next(d.message for d in defects if d.code == "reviewer_warning") + + assert message.startswith("[critical] Review phase missing") + assert "Remediation:" in message + + +class TestReviewAuditCoverage: + def test_pii_classifier_signal_staffs_audit_before_validation( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + monkeypatch.delenv("BATON_PLANNER_HARD_GATE", raising=False) + + planner = IntelligentPlanner(classifier=DataClassifier()) + + plan = planner.create_plan( + "Build a customer profile export that includes user email address " + "and SSN fields." + ) + + audit_phase = next( + phase + for phase in plan.phases + if phase.name.lower().split()[-1] == "audit" + ) + assert any(step.agent_name == "auditor" for step in audit_phase.steps) + + def test_high_risk_plan_without_review_phase_is_critical(self) -> None: + draft = _draft_with_phase(risk=RiskLevel.HIGH) + + defects = ValidationStage()._detect_defects(draft) + + assert any(d.code == "review_missing" for d in defects) + message = next(d.message for d in defects if d.code == "review_missing") + assert "risk=HIGH" in message + assert "Review" in message + assert "Remediation:" in message + + def test_compliance_plan_without_audit_phase_is_critical(self) -> None: + draft = _draft_with_phase( + task_summary="Update GDPR export compliance workflow", + agent_name="auditor", + ) + draft.resolved_agents = ["backend-engineer", "auditor"] + + defects = ValidationStage()._detect_defects(draft) + + assert any(d.code == "audit_missing" for d in defects) + message = next(d.message for d in defects if d.code == "audit_missing") + assert "auditor" in message + assert "Audit" in message + assert "Remediation:" in message + + def test_regulated_classification_without_audit_phase_is_critical(self) -> None: + draft = _draft_with_phase( + task_summary="Update FERPA student records export workflow", + risk=RiskLevel.HIGH, + agent_name="backend-engineer", + ) + draft.classification = ClassificationResult( + risk_level=RiskLevel.HIGH, + guardrail_preset="Regulated Data", + signals_found=["regulated:ferpa"], + confidence="low", + ) + + defects = ValidationStage()._detect_defects(draft) + + assert any(d.code == "audit_missing" for d in defects) + message = next(d.message for d in defects if d.code == "audit_missing") + assert "Regulated Data" in message + assert "Audit" in message + assert "Remediation:" in message + + def test_review_phase_without_reviewer_is_missing_coverage(self) -> None: + draft = _draft_with_phase(risk=RiskLevel.HIGH) + draft.plan_phases.append( + PlanPhase( + phase_id=2, + name="Review", + steps=[ + PlanStep( + step_id="2.1", + agent_name="architect", + task_description="Review the implementation.", + ) + ], + ) + ) + + defects = ValidationStage()._detect_defects(draft) + + assert any(d.code == "review_missing" for d in defects) + + def test_audit_phase_without_auditor_is_missing_coverage(self) -> None: + draft = _draft_with_phase( + task_summary="Update GDPR export compliance workflow", + agent_name="backend-engineer", + ) + draft.resolved_agents = ["backend-engineer", "auditor"] + draft.plan_phases.append( + PlanPhase( + phase_id=2, + name="Audit", + steps=[ + PlanStep( + step_id="2.1", + agent_name="code-reviewer", + task_description="Audit the implementation.", + ) + ], + ) + ) + + defects = ValidationStage()._detect_defects(draft) + + assert any(d.code == "audit_missing" for d in defects) + + def test_reviewer_agent_in_implementation_phase_without_review_is_mismatch( + self, + ) -> None: + draft = _draft_with_phase(agent_name="code-reviewer") + draft.resolved_agents = ["backend-engineer", "code-reviewer"] + + defects = ValidationStage()._detect_defects(draft) + + assert any(d.code == "agent_phase_mismatch" for d in defects) + message = next(d.message for d in defects if d.code == "agent_phase_mismatch") + assert "code-reviewer" in message + assert "Review" in message + assert "Remediation:" in message diff --git a/tests/snapshots/plans/compliance-audit.json b/tests/snapshots/plans/compliance-audit.json new file mode 100644 index 00000000..0af9d9ae --- /dev/null +++ b/tests/snapshots/plans/compliance-audit.json @@ -0,0 +1,91 @@ +{ + "budget_tier": "standard", + "complexity": "medium", + "phases": [ + { + "approval_required": true, + "name": "Design", + "phase_id": 1, + "steps": [ + { + "agent": "architect", + "depends_on": [], + "step_id": "1.1", + "step_type": "consulting", + "team": [] + }, + { + "agent": "architect", + "depends_on": [], + "step_id": "1.2", + "step_type": "planning", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Implement", + "phase_id": 2, + "steps": [ + { + "agent": "team", + "depends_on": [], + "step_id": "2.1", + "step_type": "developing", + "team": [ + { + "agent": "backend-engineer", + "member_id": "2.1.a", + "role": "lead" + } + ] + } + ] + }, + { + "approval_required": false, + "name": "Test", + "phase_id": 3, + "steps": [ + { + "agent": "test-engineer", + "depends_on": [], + "step_id": "3.1", + "step_type": "testing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Review", + "phase_id": 4, + "steps": [ + { + "agent": "code-reviewer", + "depends_on": [], + "step_id": "4.1", + "step_type": "reviewing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Audit", + "phase_id": 5, + "steps": [ + { + "agent": "auditor", + "depends_on": [], + "step_id": "5.1", + "step_type": "reviewing", + "team": [] + } + ] + } + ], + "risk_level": "HIGH", + "task_type": "new-feature" +} diff --git a/tests/snapshots/plans/compound-multi-concern.json b/tests/snapshots/plans/compound-multi-concern.json new file mode 100644 index 00000000..10abddb1 --- /dev/null +++ b/tests/snapshots/plans/compound-multi-concern.json @@ -0,0 +1,38 @@ +{ + "budget_tier": "lean", + "complexity": "light", + "phases": [ + { + "approval_required": false, + "name": "Implement", + "phase_id": 1, + "steps": [ + { + "agent": "frontend-engineer", + "depends_on": [], + "step_id": "1.1", + "step_type": "developing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Review", + "phase_id": 2, + "steps": [ + { + "agent": "code-reviewer", + "depends_on": [ + "1.1" + ], + "step_id": "2.1", + "step_type": "reviewing", + "team": [] + } + ] + } + ], + "risk_level": "LOW", + "task_type": "test" +} diff --git a/tests/snapshots/plans/direct-light.json b/tests/snapshots/plans/direct-light.json new file mode 100644 index 00000000..4e8348b6 --- /dev/null +++ b/tests/snapshots/plans/direct-light.json @@ -0,0 +1,33 @@ +{ + "budget_tier": "lean", + "complexity": "light", + "phases": [ + { + "approval_required": false, + "name": "Implement", + "phase_id": 1, + "steps": [ + { + "agent": "team", + "depends_on": [], + "step_id": "1.1", + "step_type": "developing", + "team": [ + { + "agent": "backend-engineer", + "member_id": "1.1.a", + "role": "lead" + }, + { + "agent": "test-engineer", + "member_id": "1.1.b", + "role": "implementer" + } + ] + } + ] + } + ], + "risk_level": "LOW", + "task_type": "bug-fix" +} diff --git a/tests/snapshots/plans/high-risk-security.json b/tests/snapshots/plans/high-risk-security.json new file mode 100644 index 00000000..fd107452 --- /dev/null +++ b/tests/snapshots/plans/high-risk-security.json @@ -0,0 +1,71 @@ +{ + "budget_tier": "standard", + "complexity": "medium", + "phases": [ + { + "approval_required": true, + "name": "Design", + "phase_id": 1, + "steps": [ + { + "agent": "architect", + "depends_on": [], + "step_id": "1.1", + "step_type": "consulting", + "team": [] + }, + { + "agent": "architect", + "depends_on": [], + "step_id": "1.2", + "step_type": "planning", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Implement", + "phase_id": 2, + "steps": [ + { + "agent": "backend-engineer", + "depends_on": [], + "step_id": "2.1", + "step_type": "developing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Test", + "phase_id": 3, + "steps": [ + { + "agent": "test-engineer", + "depends_on": [], + "step_id": "3.1", + "step_type": "testing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Review", + "phase_id": 4, + "steps": [ + { + "agent": "code-reviewer", + "depends_on": [], + "step_id": "4.1", + "step_type": "reviewing", + "team": [] + } + ] + } + ], + "risk_level": "HIGH", + "task_type": "refactor" +} diff --git a/tests/snapshots/plans/investigative-bug.json b/tests/snapshots/plans/investigative-bug.json new file mode 100644 index 00000000..a77f637b --- /dev/null +++ b/tests/snapshots/plans/investigative-bug.json @@ -0,0 +1,64 @@ +{ + "budget_tier": "lean", + "complexity": "medium", + "phases": [ + { + "approval_required": false, + "name": "Design", + "phase_id": 1, + "steps": [ + { + "agent": "backend-engineer", + "depends_on": [], + "step_id": "1.1", + "step_type": "developing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Implement", + "phase_id": 2, + "steps": [ + { + "agent": "backend-engineer", + "depends_on": [], + "step_id": "2.1", + "step_type": "developing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Test", + "phase_id": 3, + "steps": [ + { + "agent": "backend-engineer", + "depends_on": [], + "step_id": "3.1", + "step_type": "developing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Review", + "phase_id": 4, + "steps": [ + { + "agent": "code-reviewer", + "depends_on": [], + "step_id": "4.1", + "step_type": "reviewing", + "team": [] + } + ] + } + ], + "risk_level": "LOW", + "task_type": "bugfix" +} diff --git a/tests/snapshots/plans/knowledge-heavy.json b/tests/snapshots/plans/knowledge-heavy.json new file mode 100644 index 00000000..59c5dc86 --- /dev/null +++ b/tests/snapshots/plans/knowledge-heavy.json @@ -0,0 +1,38 @@ +{ + "budget_tier": "standard", + "complexity": "light", + "phases": [ + { + "approval_required": false, + "name": "Implement", + "phase_id": 1, + "steps": [ + { + "agent": "team-lead", + "depends_on": [], + "step_id": "1.1", + "step_type": "developing", + "team": [] + } + ] + }, + { + "approval_required": false, + "name": "Review", + "phase_id": 2, + "steps": [ + { + "agent": "code-reviewer", + "depends_on": [ + "1.1" + ], + "step_id": "2.1", + "step_type": "reviewing", + "team": [] + } + ] + } + ], + "risk_level": "HIGH", + "task_type": "test" +} diff --git a/tests/test_api_executions.py b/tests/test_api_executions.py index 01468c7f..6bba2e05 100644 --- a/tests/test_api_executions.py +++ b/tests/test_api_executions.py @@ -17,7 +17,13 @@ from fastapi.testclient import TestClient # noqa: E402 from agent_baton.api.server import create_app # noqa: E402 -from agent_baton.models.execution import MachinePlan, PlanGate, PlanPhase, PlanStep # noqa: E402 +from agent_baton.models.execution import ( # noqa: E402 + MachinePlan, + PlanGate, + PlanPhase, + PlanStep, + TeamMember, +) # --------------------------------------------------------------------------- @@ -56,6 +62,104 @@ def make_test_plan(task_id: str = "test-task") -> MachinePlan: ) +def make_plan_unlocking_team(task_id: str = "team-unlock-task") -> MachinePlan: + return MachinePlan( + task_id=task_id, + task_summary="Unlock a team step after a solo setup step", + phases=[ + PlanPhase( + phase_id=0, + name="Phase 1", + steps=[ + PlanStep( + step_id="1.1", + agent_name="test-agent", + task_description="Prepare inputs", + ), + PlanStep( + step_id="1.2", + agent_name="team", + task_description="Run team implementation", + depends_on=["1.1"], + team=[ + TeamMember( + member_id="1.2.a", + agent_name="backend-engineer", + role="implementer", + task_description="Implement the service", + model="sonnet", + ), + ], + ), + ], + ), + ], + ) + + +def make_plan_with_immediate_team_wave(task_id: str = "team-immediate-task") -> MachinePlan: + return MachinePlan( + task_id=task_id, + task_summary="Dispatch a solo step and a team step in the first wave", + phases=[ + PlanPhase( + phase_id=0, + name="Phase 1", + steps=[ + PlanStep( + step_id="1.1", + agent_name="test-agent", + task_description="Prepare inputs", + ), + PlanStep( + step_id="1.2", + agent_name="team", + task_description="Run team implementation", + team=[ + TeamMember( + member_id="1.2.a", + agent_name="backend-engineer", + role="implementer", + task_description="Implement the service", + model="sonnet", + ), + ], + ), + ], + ), + ], + ) + + +def make_plan_starting_with_team_step(task_id: str = "team-first-task") -> MachinePlan: + return MachinePlan( + task_id=task_id, + task_summary="Dispatch a team step as the first action", + phases=[ + PlanPhase( + phase_id=0, + name="Phase 1", + steps=[ + PlanStep( + step_id="1.1", + agent_name="team", + task_description="Run team implementation first", + team=[ + TeamMember( + member_id="1.1.a", + agent_name="backend-engineer", + role="implementer", + task_description="Implement the service", + model="sonnet", + ), + ], + ), + ], + ), + ], + ) + + def start_execution(client: TestClient, task_id: str = "test-task") -> dict: """Helper: start an execution with an inline plan and return the response body.""" plan = make_test_plan(task_id=task_id) @@ -70,6 +174,43 @@ def start_execution(client: TestClient, task_id: str = "test-task") -> dict: class TestStartExecution: + def test_strict_unknown_team_backend_returns_500_when_first_step_is_team( + self, + app, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("BATON_TEAMS_BACKEND", "not-real") + monkeypatch.setenv("BATON_TEAMS_BACKEND_STRICT", "1") + plan = make_plan_starting_with_team_step(task_id="strict-team-first") + client = TestClient(app, raise_server_exceptions=False) + + r = client.post("/api/v1/executions", json={"plan": plan.to_dict()}) + + assert r.status_code == 500 + assert "Unknown BATON_TEAMS_BACKEND" in r.json()["detail"] + + status = client.get("/api/v1/executions/strict-team-first") + assert status.status_code == 200 + assert status.json()["status"] == "failed" + + def test_strict_unknown_team_backend_returns_500_on_initial_batch( + self, + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("BATON_TEAMS_BACKEND", "not-real") + monkeypatch.setenv("BATON_TEAMS_BACKEND_STRICT", "1") + plan = make_plan_with_immediate_team_wave(task_id="strict-team-start") + + r = client.post("/api/v1/executions", json={"plan": plan.to_dict()}) + + assert r.status_code == 500 + assert "Unknown BATON_TEAMS_BACKEND" in r.json()["detail"] + + status = client.get("/api/v1/executions/strict-team-start") + assert status.status_code == 200 + assert status.json()["status"] == "failed" + def test_inline_plan_returns_201(self, client: TestClient) -> None: plan = make_test_plan() r = client.post("/api/v1/executions", json={"plan": plan.to_dict()}) @@ -174,6 +315,55 @@ def test_record_returns_next_actions(self, client: TestClient) -> None: ) assert "next_actions" in r.json() + def test_record_surfaces_strict_unknown_team_backend( + self, + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("BATON_TEAMS_BACKEND", "not-real") + monkeypatch.setenv("BATON_TEAMS_BACKEND_STRICT", "1") + plan = make_plan_unlocking_team(task_id="strict-team") + r = client.post("/api/v1/executions", json={"plan": plan.to_dict()}) + assert r.status_code == 201, r.text + + r = client.post( + "/api/v1/executions/strict-team/record", + json={"step_id": "1.1", "agent": "test-agent", "status": "complete"}, + ) + + assert r.status_code == 500 + assert "Unknown BATON_TEAMS_BACKEND" in r.json()["detail"] + + def test_unknown_backend_mid_run_keeps_execution_running( + self, + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Mid-run strict backend errors are recoverable: status stays running. + + Start stamps failed (nothing has run, re-POST the plan); mid-run the + misconfiguration is fixable via env, so completed work must remain + resumable. See _collect_next_actions in api/routes/executions.py. + """ + monkeypatch.delenv("BATON_TEAMS_BACKEND", raising=False) + monkeypatch.delenv("BATON_TEAMS_BACKEND_STRICT", raising=False) + plan = make_plan_unlocking_team(task_id="strict-team-midrun") + r = client.post("/api/v1/executions", json={"plan": plan.to_dict()}) + assert r.status_code == 201, r.text + + monkeypatch.setenv("BATON_TEAMS_BACKEND", "not-real") + monkeypatch.setenv("BATON_TEAMS_BACKEND_STRICT", "1") + r = client.post( + "/api/v1/executions/strict-team-midrun/record", + json={"step_id": "1.1", "agent": "test-agent", "status": "complete"}, + ) + assert r.status_code == 500 + assert "Unknown BATON_TEAMS_BACKEND" in r.json()["detail"] + + status = client.get("/api/v1/executions/strict-team-midrun") + assert status.status_code == 200 + assert status.json()["status"] == "running" + def test_record_on_nonexistent_task_returns_404(self, client: TestClient) -> None: r = client.post( "/api/v1/executions/no-such-task/record", diff --git a/tests/test_api_plans.py b/tests/test_api_plans.py index c0624ef9..f96883ab 100644 --- a/tests/test_api_plans.py +++ b/tests/test_api_plans.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest +import yaml fastapi = pytest.importorskip("fastapi") from fastapi.testclient import TestClient # noqa: E402 @@ -45,9 +46,47 @@ def make_test_plan(task_id: str = "test-task") -> MachinePlan: gate=PlanGate(gate_type="test", command="pytest"), ), ], + plan_diagnostics={ + "task_type": "feature", + "knowledge_packs_loaded": 0, + "attachments_selected": 0, + }, ) +def _write_project_knowledge_pack(project_root: Path) -> None: + pack_dir = project_root / ".claude" / "knowledge" / "project-rules" + pack_dir.mkdir(parents=True) + (pack_dir / "knowledge.yaml").write_text( + yaml.dump( + { + "name": "project-rules", + "description": "Project-specific rules", + "tags": ["architecture"], + "target_agents": ["architect", "backend-engineer"], + "default_delivery": "reference", + } + ), + encoding="utf-8", + ) + (pack_dir / "architecture.md").write_text( + "---\n" + "name: architecture\n" + "description: Architecture guide\n" + "tags: [architecture]\n" + "---\n" + + ("x" * 400), + encoding="utf-8", + ) + + +def _sandbox_empty_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + home = tmp_path / "fake-home" + home.mkdir() + monkeypatch.setattr(Path, "home", lambda: home) + return home + + # =========================================================================== # POST /api/v1/plans # =========================================================================== @@ -97,6 +136,38 @@ def test_total_steps_matches_phases(self, client: TestClient) -> None: computed = sum(len(p["steps"]) for p in body["phases"]) assert body["total_steps"] == computed + def test_create_plan_response_includes_plan_diagnostics(self, client: TestClient) -> None: + r = client.post( + "/api/v1/plans", + json={"description": "Design the architecture for a new feature"}, + ) + body = r.json() + + assert isinstance(body["plan_diagnostics"], dict) + assert "knowledge_packs_loaded" in body["plan_diagnostics"] + assert "phase_count" in body["plan_diagnostics"] + assert "selected_agents" in body["plan_diagnostics"] + + def test_create_plan_uses_request_project_path_for_knowledge_loading( + self, client: TestClient, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _sandbox_empty_home(tmp_path, monkeypatch) + project_root = tmp_path / "project-root" + _write_project_knowledge_pack(project_root) + + r = client.post( + "/api/v1/plans", + json={ + "description": "Design the architecture for a new feature", + "project_path": str(project_root), + }, + ) + body = r.json() + + assert r.status_code == 201 + assert body["plan_diagnostics"]["knowledge_packs_loaded"] == 1 + assert body["plan_diagnostics"]["attachments_selected"] > 0 + # =========================================================================== # GET /api/v1/plans/{plan_id} @@ -124,3 +195,12 @@ def test_active_plan_returns_correct_id(self, client: TestClient) -> None: r = client.get("/api/v1/plans/my-plan-123") body = r.json() assert body["plan_id"] == "my-plan-123" + + def test_active_plan_returns_plan_diagnostics(self, client: TestClient) -> None: + plan = make_test_plan(task_id="diagnostics-plan") + client.post("/api/v1/executions", json={"plan": plan.to_dict()}) + + r = client.get("/api/v1/plans/diagnostics-plan") + body = r.json() + + assert body["plan_diagnostics"] == plan.plan_diagnostics diff --git a/tests/test_api_pmo.py b/tests/test_api_pmo.py index 0b03f313..9c526e26 100644 --- a/tests/test_api_pmo.py +++ b/tests/test_api_pmo.py @@ -31,6 +31,7 @@ from __future__ import annotations from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock import pytest @@ -44,7 +45,12 @@ get_pmo_scanner, get_pmo_store, ) +from agent_baton.api.routes import pmo as pmo_routes # noqa: E402 from agent_baton.api.server import create_app # noqa: E402 +from agent_baton.core.engine.planning.stages.validation import ( # noqa: E402 + PlanDefect, + PlanQualityError, +) from agent_baton.core.events.bus import EventBus # noqa: E402 from agent_baton.core.events.events import step_completed, task_completed # noqa: E402 from agent_baton.core.pmo.scanner import PmoScanner # noqa: E402 @@ -115,6 +121,21 @@ def _signal_to_plan(signal_id: str, project_id: str) -> MachinePlan | None: return stub +def _plan_quality_error() -> PlanQualityError: + defect = PlanDefect( + code="audit_missing", + severity="critical", + message=( + "Compliance plans require Audit coverage. " + "Remediation: add a terminal Audit phase with an auditor step." + ), + ) + return PlanQualityError( + "Plan blocked by ValidationStage: [critical] audit_missing", + defects=[defect], + ) + + @pytest.fixture() def store(tmp_path: Path) -> PmoStore: return _make_tmp_store(tmp_path) @@ -504,6 +525,63 @@ def test_priority_field_accepted(self, client: TestClient) -> None: ) assert r.status_code == 201 + def test_plan_quality_error_returns_structured_422( + self, client: TestClient, app + ) -> None: + _register_project(client, project_id="gate-proj", program="GATE") + app.dependency_overrides[get_forge_session]().create_plan.side_effect = ( + _plan_quality_error() + ) + + r = client.post( + "/api/v1/pmo/forge/plan", + json={ + "description": "Build a regulated data export", + "program": "GATE", + "project_id": "gate-proj", + }, + ) + + assert r.status_code == 422 + detail = r.json()["detail"] + assert detail["error"] == "plan_quality_error" + assert detail["defects"][0]["code"] == "audit_missing" + assert "Audit phase" in detail["defects"][0]["remediation"] + + def test_plan_quality_error_publishes_failed_progress_before_sentinel( + self, + client: TestClient, + app, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + session_id = "gatefail000000000000000000000000" + monkeypatch.setattr( + pmo_routes.uuid, + "uuid4", + lambda: SimpleNamespace(hex=session_id), + ) + _register_project(client, project_id="gate-prog", program="GATE") + app.dependency_overrides[get_forge_session]().create_plan.side_effect = ( + _plan_quality_error() + ) + + r = client.post( + "/api/v1/pmo/forge/plan", + json={ + "description": "Build a regulated data export", + "program": "GATE", + "project_id": "gate-prog", + }, + ) + + assert r.status_code == 422 + queue = pmo_routes._forge_progress_queues.pop(session_id) + items = [] + while not queue.empty(): + items.append(queue.get_nowait()) + assert items[-2]["stage"] == "failed" + assert items[-1] is None + # =========================================================================== # POST /api/v1/pmo/forge/approve @@ -654,6 +732,29 @@ def test_missing_description_returns_422(self, client: TestClient) -> None: ) assert r.status_code == 422 + def test_plan_quality_error_returns_structured_422( + self, client: TestClient, app + ) -> None: + _register_project(client, project_id="rggate-proj", program="RGG") + app.dependency_overrides[get_forge_session]().regenerate_plan.side_effect = ( + _plan_quality_error() + ) + + r = client.post( + "/api/v1/pmo/forge/regenerate", + json={ + "project_id": "rggate-proj", + "description": "Refine a regulated data export", + "original_plan": _minimal_plan().to_dict(), + "answers": [], + }, + ) + + assert r.status_code == 422 + detail = r.json()["detail"] + assert detail["error"] == "plan_quality_error" + assert detail["defects"][0]["code"] == "audit_missing" + # =========================================================================== # GET /api/v1/pmo/ado/search diff --git a/tests/test_archetype_decomposition.py b/tests/test_archetype_decomposition.py index 02f57baa..80f27f06 100644 --- a/tests/test_archetype_decomposition.py +++ b/tests/test_archetype_decomposition.py @@ -15,7 +15,8 @@ def services(): """Build a minimal services container from a fresh IntelligentPlanner.""" from agent_baton.core.engine.planning.planner import IntelligentPlanner - return IntelligentPlanner()._build_services() + planner = IntelligentPlanner() + return planner._build_services(knowledge_registry=planner.knowledge_registry) def _draft_for_archetype( diff --git a/tests/test_headless.py b/tests/test_headless.py index a82b4748..f8169b54 100644 --- a/tests/test_headless.py +++ b/tests/test_headless.py @@ -279,6 +279,14 @@ def test_prompt_contains_json_output_schema(self) -> None: assert "phases" in prompt assert "risk_level" in prompt + def test_prompt_contains_validation_gate_contract(self) -> None: + prompt = HeadlessClaude._build_plan_prompt("task") + assert "HIGH or CRITICAL" in prompt + assert "Review phase" in prompt + assert "Audit phase" in prompt + assert "auditor" in prompt + assert "Reviewer-class agents" in prompt + def test_all_fields_filled(self) -> None: prompt = HeadlessClaude._build_plan_prompt( description="migrate auth to OAuth", diff --git a/tests/test_install_script.py b/tests/test_install_script.py index 2a86975e..548961a0 100644 --- a/tests/test_install_script.py +++ b/tests/test_install_script.py @@ -28,6 +28,7 @@ _NOTES_FETCH_REFSPEC = "+refs/notes/*:refs/notes/*" _NOTES_PUSH_REFSPEC = "+refs/notes/*:refs/notes/*" +_AGENT_TEMPLATE_FILES = ("base-agent.md", "flavored-agent.md", "reviewer-agent.md") def _has_bash() -> bool: @@ -143,6 +144,37 @@ def _get_git_config_all(repo: Path, key: str) -> list[str]: class TestInstallScriptNotesReplication: + def test_project_install_copies_agent_starter_templates( + self, tmp_path: Path + ) -> None: + """install.sh copies templates/agents/*.md into .claude/templates/agents/.""" + repo = tmp_path / "project_templates" + repo.mkdir() + _init_git_repo_with_remote(repo) + + result = _run_install( + repo, + env_extra={ + "BATON_SKIP_GIT_NOTES_SETUP": "1", + "BATON_SKIP_BEADS_INSTALL": "1", + }, + ) + + assert result.returncode == 0, ( + f"install.sh exited {result.returncode}\n" + f"stdout: {result.stdout}\n" + f"stderr: {result.stderr}" + ) + + installed_dir = repo / ".claude" / "templates" / "agents" + source_dir = _find_install_sh().parents[1] / "templates" / "agents" + for filename in _AGENT_TEMPLATE_FILES: + installed = installed_dir / filename + source = source_dir / filename + assert installed.read_text(encoding="utf-8") == source.read_text( + encoding="utf-8" + ) + def test_fetch_refspec_added_after_install(self, tmp_path: Path) -> None: """install.sh adds +refs/notes/*:refs/notes/* to remote.origin.fetch.""" repo = tmp_path / "project" diff --git a/tests/test_install_templates_contract.py b/tests/test_install_templates_contract.py new file mode 100644 index 00000000..b53b2a85 --- /dev/null +++ b/tests/test_install_templates_contract.py @@ -0,0 +1,71 @@ +"""Contract tests for installing generated-agent starter templates.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +from agent_baton.cli.commands.distribute.install import _cmd_install + + +ROOT = Path(__file__).resolve().parents[1] +TEMPLATE_FILES = ("base-agent.md", "flavored-agent.md", "reviewer-agent.md") +INSTALLED_TEMPLATE_DIR = Path(".claude") / "templates" / "agents" + + +def _installed_template_paths() -> tuple[str, ...]: + return tuple( + f".claude/templates/agents/{filename}" for filename in TEMPLATE_FILES + ) + + +def test_baton_install_copies_agent_starter_templates_to_project_scope( + tmp_path: Path, monkeypatch +) -> None: + monkeypatch.chdir(tmp_path) + args = argparse.Namespace( + scope="project", + source=str(ROOT), + force=True, + upgrade=False, + verify=False, + ) + + _cmd_install(args) + + for filename in TEMPLATE_FILES: + installed = tmp_path / INSTALLED_TEMPLATE_DIR / filename + source = ROOT / "templates" / "agents" / filename + assert installed.read_text(encoding="utf-8") == source.read_text( + encoding="utf-8" + ) + + +def test_powershell_installer_file_list_includes_agent_starter_templates() -> None: + text = (ROOT / "scripts" / "install.ps1").read_text(encoding="utf-8") + + assert '$AgentTemplatesSrc = Join-Path $RootDir "templates" "agents"' in text + assert '$TemplateAgentTarget = Join-Path $Base "templates\\agents"' in text + assert 'Get-ChildItem "$AgentTemplatesSrc\\*.md"' in text + assert "Copy-Item $_.FullName -Destination $TemplateAgentTarget -Force" in text + + +def test_shell_installer_file_list_includes_agent_starter_templates() -> None: + text = (ROOT / "scripts" / "install.sh").read_text(encoding="utf-8") + + assert 'AGENT_TEMPLATES_SRC="$ROOT_DIR/templates/agents"' in text + assert 'TEMPLATE_AGENT_TARGET="$BASE/templates/agents"' in text + assert 'for f in "$AGENT_TEMPLATES_SRC"/*.md; do' in text + assert 'cp "$f" "$TEMPLATE_AGENT_TARGET/"' in text + + +def test_documented_starter_template_paths_match_installed_paths() -> None: + expected_paths = _installed_template_paths() + docs = ( + ROOT / "references" / "agent-authoring.md", + ROOT / "agents" / "talent-builder.md", + ) + + for doc in docs: + text = doc.read_text(encoding="utf-8") + for expected_path in expected_paths: + assert expected_path in text, f"{doc} missing {expected_path}" diff --git a/tests/test_knowledge_registry.py b/tests/test_knowledge_registry.py index 94c04d96..aec5c6e6 100644 --- a/tests/test_knowledge_registry.py +++ b/tests/test_knowledge_registry.py @@ -387,6 +387,100 @@ def test_corrupted_manifest_still_loads_pack(self, tmp_path: Path) -> None: pack = reg.get_pack("corrupted") assert pack is not None + def test_sequence_manifest_loads_pack_in_degraded_mode( + self, tmp_path: Path + ) -> None: + root = tmp_path / "knowledge" + pack_dir = root / "sequence-manifest" + pack_dir.mkdir(parents=True) + (pack_dir / "knowledge.yaml").write_text( + "- not\n- a\n- mapping\n", encoding="utf-8" + ) + _make_doc(pack_dir, "doc.md", name="doc") + + reg = KnowledgeRegistry() + count = reg.load_directory(root) + + assert count == 1 + pack = reg.get_pack("sequence-manifest") + assert pack is not None + assert reg.degraded_pack_names == {"sequence-manifest"} + + def test_non_utf8_manifest_loads_pack_in_degraded_mode( + self, tmp_path: Path + ) -> None: + root = tmp_path / "knowledge" + pack_dir = root / "legacy-manifest" + pack_dir.mkdir(parents=True) + (pack_dir / "knowledge.yaml").write_bytes( + b"name: legacy-\xe9\ndescription: legacy\n" + ) + _make_doc(pack_dir, "doc.md", name="doc") + + reg = KnowledgeRegistry() + count = reg.load_directory(root) + + assert count == 1 + pack = reg.get_pack("legacy-manifest") + assert pack is not None + assert reg.degraded_pack_names == {"legacy-manifest"} + + def test_bad_pack_does_not_block_good_packs_or_index_rebuild( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + root = tmp_path / "knowledge" + + first_good = root / "a-good" + first_good.mkdir(parents=True) + _make_manifest( + first_good, + name="first-good", + description="Reliable recovery references", + tags=["resilience"], + ) + _make_doc( + first_good, + "recovery.md", + name="recovery", + description="Reliable recovery tokens", + tags=["resilience"], + ) + + bad = root / "b-bad" + bad.mkdir() + _make_manifest(bad, name="bad") + (bad / "bad.md").write_text( + "---\n- not\n- metadata\n---\nbody\n", encoding="utf-8" + ) + + later_good = root / "c-good" + later_good.mkdir() + _make_manifest( + later_good, + name="later-good", + description="Durable indexing references", + tags=["indexing"], + ) + _make_doc( + later_good, + "index.md", + name="indexing", + description="Durable indexing signals", + tags=["indexing"], + ) + + reg = KnowledgeRegistry() + count = reg.load_directory(root) + + assert count == 2 + assert set(reg.all_packs) == {"first-good", "later-good"} + assert [doc.name for doc, _score in reg.search("reliable recovery")] + assert [doc.name for doc, _score in reg.search("durable indexing")] + assert any( + "Skipping knowledge pack" in record.message and str(bad) in record.message + for record in caplog.records + ) + # --------------------------------------------------------------------------- # TestGetDocument diff --git a/tests/test_planner_knowledge.py b/tests/test_planner_knowledge.py index e4c85b23..97ac5c17 100644 --- a/tests/test_planner_knowledge.py +++ b/tests/test_planner_knowledge.py @@ -79,6 +79,13 @@ def _make_registry(knowledge_root: Path) -> KnowledgeRegistry: return reg +def _sandbox_empty_home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + home = tmp_path / "fake-home" + home.mkdir() + monkeypatch.setattr(Path, "home", lambda: home) + return home + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -135,6 +142,7 @@ def planner_no_registry(tmp_path: Path) -> IntelligentPlanner: """IntelligentPlanner with no knowledge_registry — knowledge resolution is skipped.""" return IntelligentPlanner( team_context_root=tmp_path / "team-context", + knowledge_registry=None, ) @@ -150,9 +158,136 @@ def test_accepts_knowledge_registry(self, registry: KnowledgeRegistry, tmp_path: ) assert planner.knowledge_registry is registry - def test_default_registry_is_none(self, tmp_path: Path) -> None: + def test_default_registry_loads_project_knowledge( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + knowledge_root = tmp_path / ".claude" / "knowledge" + pack_dir = knowledge_root / "project-rules" + pack_dir.mkdir(parents=True) + _make_manifest( + pack_dir, + name="project-rules", + description="Project-specific rules", + tags=["architecture"], + target_agents=["architect", "backend-engineer"], + ) + _make_doc( + pack_dir, + "architecture.md", + name="architecture", + tags=["architecture"], + ) + monkeypatch.chdir(tmp_path) + planner = IntelligentPlanner(team_context_root=tmp_path / "tc") + assert planner.knowledge_registry is not None + assert planner.knowledge_registry.get_pack("project-rules") is not None + + def test_explicit_none_preserves_registry_opt_out( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + knowledge_root = tmp_path / ".claude" / "knowledge" + pack_dir = knowledge_root / "project-rules" + pack_dir.mkdir(parents=True) + _make_manifest( + pack_dir, + name="project-rules", + description="Project-specific rules", + tags=["architecture"], + target_agents=["architect", "backend-engineer"], + ) + _make_doc( + pack_dir, + "architecture.md", + name="architecture", + tags=["architecture"], + ) + monkeypatch.chdir(tmp_path) + + planner = IntelligentPlanner( + team_context_root=tmp_path / "tc", + knowledge_registry=None, + ) + assert planner.knowledge_registry is None + plan = planner.create_plan("Design the architecture for a new endpoint") + for step in plan.all_steps: + assert step.knowledge == [] + + def test_create_plan_uses_project_root_for_auto_managed_registry( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _sandbox_empty_home(tmp_path, monkeypatch) + project_root = tmp_path / "project-root" + knowledge_root = project_root / ".claude" / "knowledge" + pack_dir = knowledge_root / "project-rules" + pack_dir.mkdir(parents=True) + _make_manifest( + pack_dir, + name="project-rules", + description="Project-specific rules", + tags=["architecture"], + target_agents=["architect", "backend-engineer"], + ) + _make_doc( + pack_dir, + "architecture.md", + name="architecture", + description="Architecture guide", + tags=["architecture"], + ) + outside_cwd = tmp_path / "outside-cwd" + outside_cwd.mkdir() + monkeypatch.chdir(outside_cwd) + + planner = IntelligentPlanner(team_context_root=tmp_path / "tc") + plan = planner.create_plan( + "Design the architecture for a new endpoint", + project_root=project_root, + ) + + diagnostics = plan.plan_diagnostics + assert diagnostics["knowledge_packs_loaded"] == 1 + assert diagnostics["docs_indexed"] == 1 + assert diagnostics["attachments_selected"] > 0 + + def test_explicit_none_ignores_project_root_knowledge( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + project_root = tmp_path / "project-root" + knowledge_root = project_root / ".claude" / "knowledge" + pack_dir = knowledge_root / "project-rules" + pack_dir.mkdir(parents=True) + _make_manifest( + pack_dir, + name="project-rules", + description="Project-specific rules", + tags=["architecture"], + target_agents=["architect", "backend-engineer"], + ) + _make_doc( + pack_dir, + "architecture.md", + name="architecture", + tags=["architecture"], + ) + outside_cwd = tmp_path / "outside-cwd" + outside_cwd.mkdir() + monkeypatch.chdir(outside_cwd) + + planner = IntelligentPlanner( + team_context_root=tmp_path / "tc", + knowledge_registry=None, + ) + plan = planner.create_plan( + "Design the architecture for a new endpoint", + project_root=project_root, + ) + + assert plan.plan_diagnostics["knowledge_packs_loaded"] == 0 + assert plan.plan_diagnostics["attachments_selected"] == 0 + for step in plan.all_steps: + assert step.knowledge == [] # --------------------------------------------------------------------------- @@ -258,6 +393,75 @@ def test_knowledge_resolution_does_not_break_normal_plan_structure( assert plan.task_id != "" assert plan.risk_level in ("LOW", "MEDIUM", "HIGH", "CRITICAL") + +# --------------------------------------------------------------------------- +# Tests: plan diagnostics +# --------------------------------------------------------------------------- + +class TestPlanKnowledgeDiagnostics: + def test_no_knowledge_packs_reports_zero_loaded( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _sandbox_empty_home(tmp_path, monkeypatch) + monkeypatch.chdir(tmp_path) + planner = IntelligentPlanner(team_context_root=tmp_path / "team-context") + + plan = planner.create_plan("Build a small validation helper") + + diagnostics = plan.plan_diagnostics + assert diagnostics["knowledge_packs_loaded"] == 0 + assert diagnostics["docs_indexed"] == 0 + assert diagnostics["attachments_selected"] == 0 + assert diagnostics["degraded_packs"] == [] + assert diagnostics["phase_count"] == len(plan.phases) + assert diagnostics["gate_count"] == sum(1 for p in plan.phases if p.gate) + assert diagnostics["validation_warning_count"] >= 0 + + def test_loaded_knowledge_packs_and_attachments_are_reported( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + _sandbox_empty_home(tmp_path, monkeypatch) + knowledge_root = tmp_path / ".claude" / "knowledge" + pack_dir = knowledge_root / "architecture-rules" + pack_dir.mkdir(parents=True) + _make_manifest( + pack_dir, + name="architecture-rules", + description="Architecture rules", + tags=["architecture"], + target_agents=["architect", "backend-engineer"], + ) + _make_doc( + pack_dir, + "architecture.md", + name="architecture", + description="Architecture decisions", + tags=["architecture"], + ) + monkeypatch.chdir(tmp_path) + planner = IntelligentPlanner(team_context_root=tmp_path / "team-context") + + plan = planner.create_plan("Design the architecture for a new endpoint") + + diagnostics = plan.plan_diagnostics + assert diagnostics["knowledge_packs_loaded"] == 1 + assert diagnostics["docs_indexed"] == 1 + assert diagnostics["attachments_selected"] > 0 + assert diagnostics["knowledge_attachment_count"] == diagnostics["attachments_selected"] + assert diagnostics["degraded_packs"] == [] + assert diagnostics["selected_agents"] + + def test_explain_plan_includes_concise_diagnostics_block( + self, planner_with_registry: IntelligentPlanner + ) -> None: + plan = planner_with_registry.create_plan("Design the architecture for an API") + + explanation = planner_with_registry.explain_plan(plan) + + assert "## Plan Diagnostics" in explanation + assert "knowledge_packs_loaded" in explanation + assert "attachments_selected" in explanation + def test_knowledge_dedup_within_step( self, planner_with_registry: IntelligentPlanner ) -> None: diff --git a/tests/test_pmo_forge.py b/tests/test_pmo_forge.py index 9f015446..0deed65c 100644 --- a/tests/test_pmo_forge.py +++ b/tests/test_pmo_forge.py @@ -3,15 +3,20 @@ import json from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest +from agent_baton.core.engine.planner import IntelligentPlanner +from agent_baton.core.govern.classifier import DataClassifier +from agent_baton.core.orchestration.knowledge_registry import KnowledgeRegistry from agent_baton.core.pmo.forge import ForgeSession from agent_baton.core.pmo.store import PmoStore from agent_baton.core.runtime.headless import HeadlessClaude, HeadlessConfig +from agent_baton.core.engine.planning.stages.validation import PlanQualityError from agent_baton.models.execution import MachinePlan, PlanPhase, PlanStep -from agent_baton.models.pmo import PmoProject, PmoSignal +from agent_baton.models.pmo import InterviewAnswer, PmoProject, PmoSignal # --------------------------------------------------------------------------- @@ -83,11 +88,229 @@ def _forge(planner: object, store: PmoStore) -> ForgeSession: return ForgeSession(planner=planner, store=store, headless=disabled_headless) +class _AvailableHeadless: + def __init__(self, plan: MachinePlan) -> None: + self.is_available = True + self._plan = plan + + async def generate_plan(self, **kwargs) -> MachinePlan: # noqa: ARG002 + return self._plan + + +def _planner_with_reviewer_warning( + monkeypatch: pytest.MonkeyPatch, + warning: str = "[critical] reviewer requires additional validation coverage", +) -> IntelligentPlanner: + planner = IntelligentPlanner() + + def _review(**kwargs): # noqa: ARG001 + return SimpleNamespace( + warnings=[warning], + splits_applied=0, + source="test-reviewer", + ) + + monkeypatch.setattr(planner._plan_reviewer, "review", _review) + return planner + + # --------------------------------------------------------------------------- # create_plan # --------------------------------------------------------------------------- class TestCreatePlan: + def test_headless_reviewer_warning_raises_plan_quality_error( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + monkeypatch.delenv("BATON_PLANNER_HARD_GATE", raising=False) + + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = _planner_with_reviewer_warning(monkeypatch) + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(_plan(task_id="headless-review-warning")), + ) + + with pytest.raises(PlanQualityError, match="reviewer_warning"): + forge.create_plan( + description="Implement the login fix", + program="NDS", + project_id="nds", + ) + + def test_headless_invalid_plan_raises_plan_quality_error(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + monkeypatch.delenv("BATON_PLANNER_HARD_GATE", raising=False) + + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = IntelligentPlanner() + invalid_plan = _plan( + task_id="headless-invalid-task", + task_summary="Invalid headless plan", + phases=[ + PlanPhase( + phase_id=0, + name="Implement", + steps=[ + PlanStep( + step_id="1.1", + agent_name="code-reviewer", + task_description="Review code during implementation", + ) + ], + ) + ], + ) + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(invalid_plan), + ) + + with pytest.raises(PlanQualityError): + forge.create_plan( + description="Implement a change with an invalid reviewer assignment", + program="NDS", + project_id="nds", + ) + + def test_headless_plan_uses_planner_effective_knowledge_registry( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + registry = KnowledgeRegistry() + planner = IntelligentPlanner(knowledge_registry=registry) + seen_registries: list[object] = [] + original_build_services = planner._build_services + + def _build_services(*, knowledge_registry): + seen_registries.append(knowledge_registry) + return original_build_services(knowledge_registry=knowledge_registry) + + monkeypatch.setattr(planner, "_build_services", _build_services) + headless_plan = _plan(task_id="headless-custom-registry") + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(headless_plan), + ) + + result = forge.create_plan( + description="Implement the login fix", + program="NDS", + project_id="nds", + ) + + assert result is headless_plan + assert seen_registries == [registry] + + def test_headless_validation_syncs_canonical_budget_tier( + self, + tmp_path: Path, + ) -> None: + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = IntelligentPlanner() + headless_plan = _plan(task_id="headless-budget-tier") + headless_plan.task_type = "new-feature" + headless_plan.budget_tier = "full" + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(headless_plan), + ) + + result = forge.create_plan( + description="Implement the login fix", + program="NDS", + project_id="nds", + ) + + assert result is headless_plan + assert result.budget_tier == "lean" + + def test_headless_validation_derives_server_side_classification( + self, + tmp_path: Path, + ) -> None: + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = IntelligentPlanner(classifier=DataClassifier()) + headless_plan = _plan( + task_id="headless-pii-parity", + task_summary=( + "Build a customer export containing email address and SSN fields." + ), + phases=[ + PlanPhase(phase_id=1, name="Implement", steps=[_step()]), + PlanPhase( + phase_id=2, + name="Review", + steps=[ + PlanStep( + step_id="2.1", + agent_name="code-reviewer", + task_description="Review high-risk data handling.", + depends_on=["1.1"], + ) + ], + ), + PlanPhase( + phase_id=3, + name="Audit", + steps=[ + PlanStep( + step_id="3.1", + agent_name="auditor", + task_description="Audit regulated data handling.", + depends_on=["2.1"], + ) + ], + ), + ], + ) + headless_plan.risk_level = "LOW" + headless_plan.classification_signals = None + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(headless_plan), + ) + + result = forge.create_plan( + description=headless_plan.task_summary, + program="NDS", + project_id="nds", + ) + + assert result is headless_plan + assert result.risk_level == "HIGH" + assert result.classification_signals is not None + signals = json.loads(result.classification_signals) + assert signals["guardrail_preset"] == "Regulated Data" + assert "pii:ssn" in signals["signals"] + def test_delegates_to_planner(self, tmp_path: Path): store = _store(tmp_path) project = _project(tmp_path) @@ -185,6 +408,181 @@ def test_project_root_is_none_when_project_not_found(self, tmp_path: Path): project_root_arg = call_kwargs.kwargs.get("project_root") assert project_root_arg is None + def test_headless_plan_gets_plan_diagnostics(self, tmp_path: Path): + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = _mock_planner() + headless_plan = _plan(task_id="headless-task", task_summary="Headless plan") + headless_plan.plan_diagnostics = {} + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(headless_plan), + ) + + result = forge.create_plan( + description="Design the architecture for a new feature", + program="NDS", + project_id="nds", + ) + + assert result is headless_plan + assert result.classification_source == "headless-claude" + assert result.plan_diagnostics["classification_source"] == "headless-claude" + assert result.plan_diagnostics["phase_count"] == len(result.phases) + assert result.plan_diagnostics["selected_agents"] == ["backend-engineer"] + assert "knowledge_packs_loaded" in result.plan_diagnostics + assert "attachments_selected" in result.plan_diagnostics + planner.create_plan.assert_not_called() + + def test_headless_reviewer_warning_warn_only_returns_plan_and_records_warning_count( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("BATON_PLANNER_WARN_ONLY", "1") + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_HARD_GATE", raising=False) + + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = _planner_with_reviewer_warning(monkeypatch) + headless_plan = _plan(task_id="headless-warn-only") + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(headless_plan), + ) + + result = forge.create_plan( + description="Implement the login fix", + program="NDS", + project_id="nds", + ) + + assert result is headless_plan + assert result.plan_diagnostics["validation_warning_count"] >= 1 + + +# --------------------------------------------------------------------------- +# regenerate_plan +# --------------------------------------------------------------------------- + +class TestRegeneratePlan: + def test_headless_reviewer_warning_raises_plan_quality_error( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + monkeypatch.delenv("BATON_PLANNER_HARD_GATE", raising=False) + + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = _planner_with_reviewer_warning(monkeypatch) + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(_plan(task_id="regen-review-warning")), + ) + + with pytest.raises(PlanQualityError, match="reviewer_warning"): + forge.regenerate_plan( + description="Refine the login fix plan", + project_id="nds", + answers=[InterviewAnswer(question_id="q-testing", answer="Add tests")], + ) + + def test_headless_invalid_regenerated_plan_raises_plan_quality_error( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ): + monkeypatch.delenv("BATON_DEV_MODE", raising=False) + monkeypatch.delenv("BATON_PLANNER_WARN_ONLY", raising=False) + monkeypatch.delenv("BATON_PLANNER_HARD_GATE", raising=False) + + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = IntelligentPlanner() + invalid_plan = _plan( + task_id="regen-invalid-task", + task_summary="Invalid regenerated plan", + phases=[ + PlanPhase( + phase_id=0, + name="Implement", + steps=[ + PlanStep( + step_id="1.1", + agent_name="code-reviewer", + task_description="Review code during implementation", + ) + ], + ) + ], + ) + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(invalid_plan), + ) + + with pytest.raises(PlanQualityError): + forge.regenerate_plan( + description="Refine the invalid reviewer plan", + project_id="nds", + answers=[ + InterviewAnswer( + question_id="q-testing", + answer="Add unit tests", + ) + ], + ) + + def test_headless_regenerated_plan_gets_plan_diagnostics(self, tmp_path: Path): + store = _store(tmp_path) + project = _project(tmp_path) + store.register_project(project) + + planner = _mock_planner() + headless_plan = _plan(task_id="regen-task", task_summary="Regenerated plan") + headless_plan.plan_diagnostics = {} + forge = ForgeSession( + planner=planner, + store=store, + headless=_AvailableHeadless(headless_plan), + ) + + result = forge.regenerate_plan( + description="Design the architecture for a new feature", + project_id="nds", + answers=[ + InterviewAnswer( + question_id="q-testing", + answer="Add unit tests", + ) + ], + ) + + assert result is headless_plan + assert result.classification_source == "headless-claude" + assert result.plan_diagnostics["classification_source"] == "headless-claude" + assert result.plan_diagnostics["phase_count"] == len(result.phases) + assert result.plan_diagnostics["selected_agents"] == ["backend-engineer"] + assert "knowledge_packs_loaded" in result.plan_diagnostics + assert "attachments_selected" in result.plan_diagnostics + planner.create_plan.assert_not_called() + # --------------------------------------------------------------------------- # save_plan